diff --git a/Cargo.lock b/Cargo.lock index 7d6b502fe55..0d4c4be392a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1631,11 +1631,13 @@ dependencies = [ "commonware-actor", "commonware-broadcast", "commonware-codec", + "commonware-coding", "commonware-conformance", "commonware-consensus", "commonware-cryptography", "commonware-formatting", "commonware-macros", + "commonware-math", "commonware-p2p", "commonware-parallel", "commonware-resolver", diff --git a/consensus/src/lib.rs b/consensus/src/lib.rs index be7bf19c888..611d3d43f75 100644 --- a/consensus/src/lib.rs +++ b/consensus/src/lib.rs @@ -274,15 +274,22 @@ stability_scope!(ALPHA, cfg(not(target_arch = "wasm32")) { /// The block type produced by the application's builder. type Block: Block; + /// Per-proposal input handed to [`propose`](Self::propose). Applications + /// that need no input set this to `()`. + type Input: Send; + /// Build a new block on top of the provided parent ancestry. If the build job fails, /// or the proposer's slot should be skipped, the implementor should return [None]. /// + /// `input` is the per-proposal input for this build. + /// /// This future may be cancelled before it completes. Implementations must be /// cancellation-safe. fn propose( &mut self, context: (E, Self::Context), ancestry: impl Ancestry, + input: Self::Input, ) -> impl Future> + Send; /// Verify a block produced by the application's proposer, relative to its ancestry. diff --git a/consensus/src/marshal/ancestry.rs b/consensus/src/marshal/ancestry.rs index 59fe6213e6d..faa8a7ae2cb 100644 --- a/consensus/src/marshal/ancestry.rs +++ b/consensus/src/marshal/ancestry.rs @@ -17,7 +17,7 @@ use std::{ }; /// A stream of blocks used by application propose and verify calls. -pub trait Ancestry: Stream> + Send + Unpin + 'static { +pub trait Ancestry: Stream> + Clone + Send + Unpin + 'static { /// Peeks at the latest block in the stream without consuming it. Returns [None] /// if the stream does not yet have a block available or has been exhausted. fn peek(&self) -> Option<&B>; @@ -27,15 +27,79 @@ pub trait Ancestry: Stream> + Send + Unpin + 'static { /// /// Blocks are yielded in iterator order and no parent fetching is performed. This is useful when /// the caller wants to bound the ancestry available to the application. -pub fn from_iter(blocks: impl IntoIterator>) -> impl Ancestry +pub fn from_iter(blocks: impl IntoIterator>) -> impl Ancestry { + BoundedAncestry { + blocks: blocks.into_iter().collect(), + } +} + +/// Prepends a fixed sequence of blocks to an existing ancestry stream. +/// +/// Blocks are yielded in iterator order before the tail is polled. +pub fn with_prefix(blocks: impl IntoIterator>, tail: S) -> impl Ancestry where B: Block, + S: Ancestry, { - BoundedAncestry { + PrefixedAncestry { blocks: blocks.into_iter().collect(), + tail, + } +} + +/// Type-erased ancestry stream that preserves cloneability. +pub struct BoxedAncestry(Box>); + +impl BoxedAncestry { + /// Erases the concrete ancestry stream type. + pub fn new(ancestry: impl Ancestry) -> Self { + Self(Box::new(ancestry)) + } +} + +impl Clone for BoxedAncestry { + fn clone(&self) -> Self { + Self(self.0.clone_box()) + } +} + +impl Unpin for BoxedAncestry {} + +impl Ancestry for BoxedAncestry { + fn peek(&self) -> Option<&B> { + self.0.peek_erased() } } +impl Stream for BoxedAncestry { + type Item = Arc; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.0).poll_next(cx) + } +} + +trait ErasedAncestry: Stream> + Send + Unpin + 'static { + fn peek_erased(&self) -> Option<&B>; + + fn clone_box(&self) -> Box>; +} + +impl ErasedAncestry for A +where + B: Block, + A: Ancestry, +{ + fn peek_erased(&self) -> Option<&B> { + Ancestry::peek(self) + } + + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} + +#[derive(Clone)] struct BoundedAncestry { blocks: VecDeque>, } @@ -56,6 +120,42 @@ impl Stream for BoundedAncestry { } } +#[derive(Clone)] +struct PrefixedAncestry { + blocks: VecDeque>, + tail: S, +} + +impl Unpin for PrefixedAncestry {} + +impl Ancestry for PrefixedAncestry +where + B: Block, + S: Ancestry, +{ + fn peek(&self) -> Option<&B> { + self.blocks + .front() + .map(Arc::as_ref) + .or_else(|| self.tail.peek()) + } +} + +impl Stream for PrefixedAncestry +where + B: Block, + S: Ancestry, +{ + type Item = Arc; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if let Some(block) = self.blocks.pop_front() { + return Poll::Ready(Some(block)); + } + Pin::new(&mut self.tail).poll_next(cx) + } +} + /// An interface for providing parent blocks. pub trait BlockProvider: Send + 'static { /// The block type the provider walks. @@ -143,6 +243,7 @@ pub struct AncestorStream { marshal: M, fetch_duration: Timed, clock: Arc, + pending_child: Option>, #[pin] pending: OptionFuture>, } @@ -181,6 +282,7 @@ impl AncestorStream { buffered, fetch_duration, clock, + pending_child: None, pending: None.into(), } } @@ -192,9 +294,35 @@ impl AncestorStream { } } +impl Clone for AncestorStream +where + M: BlockProvider + Clone, + C: Clock, +{ + fn clone(&self) -> Self { + let pending_child = self.pending_child.clone(); + let marshal = self.marshal.clone(); + let fetch_duration = self.fetch_duration.clone(); + let clock = self.clock.clone(); + let pending = pending_child + .as_ref() + .map(|child| timed_parent_fetch(&clock, &marshal, child, &fetch_duration)) + .into(); + + Self { + buffered: self.buffered.clone(), + marshal, + fetch_duration, + clock, + pending_child, + pending, + } + } +} + impl Ancestry for AncestorStream where - M: BlockProvider, + M: BlockProvider + Clone, C: Clock, { fn peek(&self) -> Option<&M::Block> { @@ -222,6 +350,7 @@ where if should_walk_parent && end_of_buffered { let future = timed_parent_fetch(this.clock, this.marshal, &block, this.fetch_duration); + *this.pending_child = Some(block.clone()); *this.pending.as_mut() = Some(future).into(); // Explicitly poll the next future to kick off the fetch. If it's already ready, @@ -230,15 +359,21 @@ where Poll::Ready(Some(Some((expected, parent)))) => { expected.assert_matches(parent.as_ref()); this.buffered.push(parent); + *this.pending_child = None; } Poll::Ready(Some(None)) => { *this.pending.as_mut() = None.into(); + *this.pending_child = None; } - Poll::Ready(None) | Poll::Pending => {} + Poll::Ready(None) => { + *this.pending_child = None; + } + Poll::Pending => {} } } else if !should_walk_parent { // No more parents to fetch; Finish the stream. *this.pending.as_mut() = None.into(); + *this.pending_child = None; } return Poll::Ready(Some(block)); @@ -248,6 +383,7 @@ where Poll::Pending => Poll::Pending, Poll::Ready(None) | Poll::Ready(Some(None)) => { *this.pending.as_mut() = None.into(); + *this.pending_child = None; Poll::Ready(None) } Poll::Ready(Some(Some((expected, block)))) => { @@ -257,6 +393,7 @@ where if should_walk_parent { let future = timed_parent_fetch(this.clock, this.marshal, &block, this.fetch_duration); + *this.pending_child = Some(block.clone()); *this.pending.as_mut() = Some(future).into(); // Explicitly poll the next future to kick off the fetch. If it's already ready, @@ -265,15 +402,21 @@ where Poll::Ready(Some(Some((expected, parent)))) => { expected.assert_matches(parent.as_ref()); this.buffered.push(parent); + *this.pending_child = None; } Poll::Ready(Some(None)) => { *this.pending.as_mut() = None.into(); + *this.pending_child = None; + } + Poll::Ready(None) => { + *this.pending_child = None; } - Poll::Ready(None) | Poll::Pending => {} + Poll::Pending => {} } } else { // No more parents to fetch; Finish the stream. *this.pending.as_mut() = None.into(); + *this.pending_child = None; } Poll::Ready(Some(block)) @@ -294,6 +437,7 @@ mod test { histogram::{Buckets, Timed}, }, }; + use commonware_utils::{channel::oneshot, sync::Mutex}; use futures::StreamExt; #[derive(Default, Clone)] @@ -316,6 +460,40 @@ mod test { } } + type TestBlock = Block; + type ParentSubscription = oneshot::Sender>; + + #[derive(Default, Clone)] + struct PendingProvider { + subscriptions: Arc>>, + } + + impl PendingProvider { + fn subscription_count(&self) -> usize { + self.subscriptions.lock().len() + } + + fn complete_all(&self, parent: Arc>) { + let subscriptions = std::mem::take(&mut *self.subscriptions.lock()); + for subscription in subscriptions { + assert!(subscription.send(parent.clone()).is_ok()); + } + } + } + + impl BlockProvider for PendingProvider { + type Block = Block; + + fn subscribe_parent( + &self, + _block: &Self::Block, + ) -> impl Future>> + Send + 'static { + let (subscription, parent) = oneshot::channel(); + self.subscriptions.lock().push(subscription); + parent.map(Result::ok) + } + } + #[derive(Clone)] struct WrongParentProvider(Block); impl BlockProvider for WrongParentProvider { @@ -457,6 +635,36 @@ mod test { }); } + #[test] + fn test_with_prefix_peeks_tail_when_prefix_empty() { + deterministic::Runner::default().start(|_| async move { + let block = Block::new::((), Sha256Digest::EMPTY, Height::new(1), 1); + let mut ancestry = with_prefix([], from_iter([Arc::new(block.clone())])); + + assert_eq!(ancestry.peek(), Some(&block)); + assert_eq!(ancestry.next().await.as_deref(), Some(&block)); + assert_eq!(ancestry.peek(), None); + }); + } + + #[test] + fn test_with_prefix_peeks_tail_after_prefix_consumed() { + deterministic::Runner::default().start(|_| async move { + let parent = Block::new::((), Sha256Digest::EMPTY, Height::new(1), 1); + let child = Block::new::((), parent.digest(), Height::new(2), 2); + let mut ancestry = with_prefix( + [Arc::new(child.clone())], + from_iter([Arc::new(parent.clone())]), + ); + + assert_eq!(ancestry.peek(), Some(&child)); + assert_eq!(ancestry.next().await.as_deref(), Some(&child)); + assert_eq!(ancestry.peek(), Some(&parent)); + assert_eq!(ancestry.next().await.as_deref(), Some(&parent)); + assert_eq!(ancestry.peek(), None); + }); + } + #[test] fn test_yields_genesis_and_stops() { deterministic::Runner::default().start(|context| async move { @@ -471,6 +679,33 @@ mod test { }); } + #[test] + fn test_clone_preserves_pending_parent_fetch() { + deterministic::Runner::default().start(|context| async move { + let parent = Arc::new(Block::new::( + (), + Sha256Digest::EMPTY, + Height::zero(), + 0, + )); + let child = Block::new::((), parent.digest(), Height::new(1), 1); + let provider = PendingProvider::default(); + let mut stream = stream(&context, provider.clone(), [child.clone()]); + + assert_eq!(stream.next().await.as_deref(), Some(&child)); + assert_eq!(provider.subscription_count(), 1); + + let mut cloned = stream.clone(); + assert_eq!(provider.subscription_count(), 2); + provider.complete_all(parent.clone()); + + assert_eq!(stream.next().await, Some(parent.clone())); + assert_eq!(cloned.next().await, Some(parent.clone())); + assert_eq!(stream.next().await, None); + assert_eq!(cloned.next().await, None); + }); + } + #[test] fn test_empty_yields_none() { deterministic::Runner::default().start(|context| async move { diff --git a/consensus/src/marshal/coding/marshaled.rs b/consensus/src/marshal/coding/marshaled.rs index e0fa712e3a9..d731648db59 100644 --- a/consensus/src/marshal/coding/marshaled.rs +++ b/consensus/src/marshal/coding/marshaled.rs @@ -220,6 +220,7 @@ where Block = B, SigningScheme = Z::Scheme, Context = Context::PublicKey>, + Input = (), >, B: CertifiableBlock>::Context>, C: CodingScheme, @@ -645,6 +646,7 @@ where Block = B, SigningScheme = Z::Scheme, Context = Context::PublicKey>, + Input = (), >, B: CertifiableBlock>::Context>, C: CodingScheme, @@ -821,6 +823,7 @@ where consensus_context.clone(), ), ancestor_stream, + (), ) .instrument(info_span!( "marshal.coding.application.propose", @@ -1088,6 +1091,7 @@ where Block = B, SigningScheme = Z::Scheme, Context = Context::PublicKey>, + Input = (), >, B: CertifiableBlock>::Context>, C: CodingScheme, diff --git a/consensus/src/marshal/mocks/verifying.rs b/consensus/src/marshal/mocks/verifying.rs index adcd8652ca7..107b53eb1f0 100644 --- a/consensus/src/marshal/mocks/verifying.rs +++ b/consensus/src/marshal/mocks/verifying.rs @@ -71,11 +71,13 @@ where type Block = B; type Context = B::Context; type SigningScheme = S; + type Input = (); async fn propose( &mut self, _context: (deterministic::Context, Self::Context), _ancestry: impl Ancestry, + _input: Self::Input, ) -> Option { self.propose_result.clone() } @@ -126,11 +128,13 @@ where type Block = B; type Context = B::Context; type SigningScheme = S; + type Input = (); async fn propose( &mut self, _context: (deterministic::Context, Self::Context), _ancestry: impl Ancestry, + _input: Self::Input, ) -> Option { None } diff --git a/consensus/src/marshal/standard/deferred.rs b/consensus/src/marshal/standard/deferred.rs index 92e19dfb3b2..631a6af02d5 100644 --- a/consensus/src/marshal/standard/deferred.rs +++ b/consensus/src/marshal/standard/deferred.rs @@ -184,7 +184,13 @@ impl Deferred where E: Rng + Spawner + Metrics + Clock, S: Scheme, - A: Application>, + A: Application< + E, + Block = B, + SigningScheme = S, + Context = Context, + Input = (), + >, B: CertifiableBlock>::Context>, ES: Epocher, { @@ -463,7 +469,13 @@ impl Automaton for Deferred where E: Rng + Spawner + Metrics + Clock, S: Scheme, - A: Application>, + A: Application< + E, + Block = B, + SigningScheme = S, + Context = Context, + Input = (), + >, B: CertifiableBlock>::Context>, ES: Epocher, { @@ -627,6 +639,7 @@ where consensus_context.clone(), ), ancestor_stream, + (), ) .instrument(info_span!( "marshal.deferred.application.propose", @@ -822,7 +835,13 @@ impl CertifiableAutomaton for Deferred where E: Rng + Spawner + Metrics + Clock, S: Scheme, - A: Application>, + A: Application< + E, + Block = B, + SigningScheme = S, + Context = Context, + Input = (), + >, B: CertifiableBlock>::Context>, ES: Epocher, { diff --git a/consensus/src/marshal/standard/inline.rs b/consensus/src/marshal/standard/inline.rs index 22af8bca60c..74394521cb2 100644 --- a/consensus/src/marshal/standard/inline.rs +++ b/consensus/src/marshal/standard/inline.rs @@ -174,7 +174,13 @@ impl Inline where E: Rng + Spawner + Metrics + Clock, S: Scheme, - A: Application>, + A: Application< + E, + Block = B, + SigningScheme = S, + Context = Context, + Input = (), + >, B: Block + Clone, ES: Epocher, { @@ -218,7 +224,13 @@ impl Automaton for Inline where E: Rng + Spawner + Metrics + Clock, S: Scheme, - A: Application>, + A: Application< + E, + Block = B, + SigningScheme = S, + Context = Context, + Input = (), + >, B: Block + Clone, ES: Epocher, { @@ -349,6 +361,7 @@ where consensus_context.clone(), ), ancestor_stream, + (), ) .instrument(info_span!( "marshal.inline.application.propose", @@ -558,7 +571,13 @@ impl CertifiableAutomaton for Inline where E: Rng + Spawner + Metrics + Clock, S: Scheme, - A: Application>, + A: Application< + E, + Block = B, + SigningScheme = S, + Context = Context, + Input = (), + >, B: Block + Clone, ES: Epocher, { @@ -707,7 +726,13 @@ mod tests { where E: Rng + Spawner + Metrics + Clock, S: Scheme, - A: Application>, + A: Application< + E, + Block = B, + SigningScheme = S, + Context = Context, + Input = (), + >, B: Block + Clone, ES: crate::types::Epocher, { diff --git a/consensus/src/types.rs b/consensus/src/types.rs index df3eb3b37f2..e9c4d012586 100644 --- a/consensus/src/types.rs +++ b/consensus/src/types.rs @@ -603,6 +603,14 @@ impl FixedEpocher { let last = first.checked_add(self.0 - 1)?; Some((Height::new(first), Height::new(last))) } + + /// Returns the midpoint block height in the given epoch. + /// + /// Returns `None` if the epoch is not supported. + pub fn midpoint(&self, epoch: Epoch) -> Option { + let (first, _) = self.bounds(epoch)?; + first.get().checked_add(self.0 / 2).map(Height::new) + } } impl Epocher for FixedEpocher { diff --git a/examples/reshare/src/application/core.rs b/examples/reshare/src/application/core.rs index 332d1c157b2..b1d2b4fa2f8 100644 --- a/examples/reshare/src/application/core.rs +++ b/examples/reshare/src/application/core.rs @@ -65,11 +65,13 @@ where type Context = Context; type SigningScheme = S; type Block = Block; + type Input = (); async fn propose( &mut self, (_, context): (E, Self::Context), mut ancestry: impl Ancestry, + _: Self::Input, ) -> Option { // Fetch the parent block from the ancestry stream. let parent_block = ancestry.next().await?; diff --git a/glue/Cargo.toml b/glue/Cargo.toml index 62c13206b7f..5ccdefc4fd4 100644 --- a/glue/Cargo.toml +++ b/glue/Cargo.toml @@ -16,10 +16,12 @@ workspace = true arbitrary = { workspace = true, optional = true, features = ["derive"] } bytes.workspace = true commonware-actor.workspace = true +commonware-broadcast.workspace = true commonware-codec.workspace = true commonware-consensus.workspace = true commonware-cryptography.workspace = true commonware-macros.workspace = true +commonware-math.workspace = true commonware-p2p.workspace = true commonware-parallel.workspace = true commonware-resolver.workspace = true @@ -36,6 +38,7 @@ tracing.workspace = true [dev-dependencies] commonware-broadcast.workspace = true commonware-codec.workspace = true +commonware-coding.workspace = true commonware-conformance.workspace = true commonware-consensus = { workspace = true, features = ["mocks"] } commonware-cryptography = { workspace = true, features = ["mocks"] } @@ -54,6 +57,7 @@ arbitrary = [ "commonware-codec/arbitrary", "commonware-consensus/arbitrary", "commonware-cryptography/arbitrary", + "commonware-math/arbitrary", "commonware-p2p/arbitrary", "commonware-resolver/arbitrary", "commonware-runtime/arbitrary", diff --git a/glue/conformance.toml b/glue/conformance.toml index f56d19dc614..e6215a1659d 100644 --- a/glue/conformance.toml +++ b/glue/conformance.toml @@ -2,6 +2,38 @@ n_cases = 65536 hash = "334bc977cfa11a849de1b135da67dd6b2103608838a8a5fa2093b401ea161a78" +["commonware_glue::dkg::anchor::wire::tests::CodecConformance>"] +n_cases = 65536 +hash = "0d6e85f0938ce300477cf7c675431f01528666eea8cb6d93efc48d70f3b803ea" + +["commonware_glue::dkg::anchor::wire::tests::CodecConformance"] +n_cases = 65536 +hash = "7cf90e43e614a55614d95269547add3d92c25b31040e8f3cce3761bd74986458" + +["commonware_glue::dkg::reshare::store::conformance::CodecConformance>"] +n_cases = 65536 +hash = "9e5d122980ec0436a7d77de1342381879d52fa796eaedfd8eba08117420ab691" + +["commonware_glue::dkg::types::conformance::CodecConformance>"] +n_cases = 8192 +hash = "7b3e025094ada3f7e729d2b36e3e55f60f19e2a6267111ae0c2266579cfa2578" + +["commonware_glue::dkg::types::conformance::CodecConformance"] +n_cases = 65536 +hash = "7cf90e43e614a55614d95269547add3d92c25b31040e8f3cce3761bd74986458" + +["commonware_glue::dkg::types::conformance::CodecConformance>"] +n_cases = 65536 +hash = "c928d571587fc43b99ebab969720228ce11becca411dc36908c979fe07ab359e" + +["commonware_glue::dkg::types::conformance::CodecConformance>"] +n_cases = 65536 +hash = "240c5ef5d2a29ad2f22678bbe5f8360ae0ab628b68e792573a16dcd0d0f09c4d" + +["commonware_glue::dkg::types::conformance::CodecConformance>"] +n_cases = 8192 +hash = "89d51bef0703a3cc6ac96ae71daa453113fd55e70febcd765d0791cc7d8c612c" + ["commonware_glue::stateful::db::p2p::compact::handler::tests::conformance::CodecConformance>"] n_cases = 65536 hash = "290187801284530d0d7e82c33bb6ce975a5f4daa4b104230291cea9cffcd7686" diff --git a/glue/src/dkg/anchor/actor/discovery.rs b/glue/src/dkg/anchor/actor/discovery.rs new file mode 100644 index 00000000000..2982a3155e2 --- /dev/null +++ b/glue/src/dkg/anchor/actor/discovery.rs @@ -0,0 +1,725 @@ +use super::serving::Serving; +use crate::dkg::{ + ReshareBlock, + anchor::{ActorArtifact, Artifact, mailbox::Message, wire}, + types::{EpochInfo, Payload}, +}; +use bytes::Buf; +use commonware_actor::mailbox::Receiver as ActorReceiver; +use commonware_codec::{Decode as _, Encode as _, Error as CodecError, Read}; +use commonware_consensus::{ + Epochable, Heightable, + marshal::core::Variant, + simplex::{scheme::Scheme, types::Certificate}, + types::{Epoch, Epocher, FixedEpocher, Height}, +}; +use commonware_cryptography::Signer; +use commonware_macros::select_loop; +use commonware_p2p::{Blocker, Channel, Message as P2pMessage, Receiver, Recipients, Sender}; +use commonware_parallel::Strategy; +use commonware_runtime::{Clock, ContextCell, Metrics, Spawner}; +use commonware_utils::{ + NonZeroDuration, + channel::{fallible::OneshotExt as _, mpsc, oneshot}, +}; +use futures::future::{self, Either}; +use rand_core::CryptoRng; +use tracing::{debug, warn}; + +#[derive(Debug)] +enum BoundaryResponseError { + BlockCommitment, + Decode(CodecError), + InvalidFinalization, +} + +pub(super) struct Pending { + height: Height, + epoch: Epoch, +} + +/// The discovery phase of the anchor actor. +/// +/// Waits for subscribers, listens to the Simplex certificate channel, and uses +/// the first verifiable target finalization to fetch the previous epoch boundary +/// block. Once the boundary block yields the target epoch's public +/// [`Artifact`], discovery resolves all subscribers and can hand off to +/// [`Serving`] after marshal is attached. +pub(super) struct Discovery +where + E: Spawner + CryptoRng + Clock + Metrics, + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, + ::Signer: Signer, + T: Strategy, + B: Blocker, +{ + pub(super) context: ContextCell, + pub(super) mailbox: ActorReceiver>, + pub(super) verifier: S, + pub(super) genesis: EpochInfo<::Variant, S::PublicKey>, + pub(super) strategy: T, + pub(super) blocker: B, + pub(super) epocher: FixedEpocher, + pub(super) block_codec_config: ::Cfg, + pub(super) retry_timeout: NonZeroDuration, + pub(super) artifact: Option>, + pub(super) subscribers: Vec>>, + pub(super) pending: Option, +} + +impl Discovery +where + E: Spawner + CryptoRng + Clock + Metrics, + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, + ::Signer: Signer, + T: Strategy, + B: Blocker, +{ + /// Runs discovery until shutdown or until it can hand off to [`Serving`]. + pub(super) async fn run( + mut self, + mut certificate_receiver: mpsc::Receiver<(Channel, P2pMessage)>, + mut boundary_sender: BSE, + mut boundary_receiver: BRE, + ) where + BSE: Sender, + BRE: Receiver, + { + let mut marshal = None; + let mut deadline = self.context.current() + self.retry_timeout.get(); + + select_loop! { + self.context, + on_start => { + self.subscribers + .retain(|subscriber| !subscriber.is_closed()); + if marshal.is_some() && self.subscribers.is_empty() { + break; + } + + // Arm the retry timer only while a boundary request is outstanding. + let retry = if self.pending.is_some() && self.artifact.is_none() { + Either::Left(self.context.sleep_until(deadline)) + } else { + Either::Right(future::pending()) + }; + }, + on_stopped => { + debug!("shutdown signal received"); + return; + }, + Some(message) = self.mailbox.recv() else { + debug!("mailbox closed, shutting down"); + return; + } => match message { + Message::Subscribe { response } => self.subscribe(response), + Message::Attach { marshal: attached } => { + marshal = Some(attached); + } + }, + Some((_channel, (peer, message))) = certificate_receiver.recv() else { + debug!("certificate receiver closed, shutting down"); + return; + } => { + if self.handle_certificate(peer, message, &mut boundary_sender) { + deadline = self.context.current() + self.retry_timeout.get(); + } + }, + Ok((peer, message)) = boundary_receiver.recv() else { + debug!("boundary receiver closed, shutting down"); + return; + } => { + self.handle_boundary_response(peer, message); + }, + _ = retry => { + if let Some(epoch) = self.pending.as_ref().map(|pending| pending.epoch) { + if epoch.is_zero() { + // The genesis grace elapsed with no strictly-newer + // finalization; resolve from the locally known genesis. + let artifact = Artifact { + epoch: Epoch::zero(), + finalization: None, + info: self.genesis.clone(), + }; + self.resolve(artifact); + self.pending = None; + } else { + debug!(%epoch, "re-requesting boundary block"); + Self::request_boundary(epoch, &mut boundary_sender); + deadline = self.context.current() + self.retry_timeout.get(); + } + } + }, + } + + Serving { + context: self.context, + mailbox: self.mailbox, + marshal: marshal.expect("serving requires attached marshal"), + blocker: self.blocker, + epocher: self.epocher, + artifact: self.artifact, + } + .run(boundary_sender, boundary_receiver) + .await; + } + + fn subscribe(&mut self, response: oneshot::Sender>) { + if let Some(artifact) = &self.artifact { + response.send_lossy(artifact.clone()); + return; + } + self.subscribers.push(response); + } + + /// Handle an incoming certificate, returning whether a boundary request was + /// broadcast (so the caller can arm the retry timer). + fn handle_certificate( + &mut self, + peer: S::PublicKey, + message: impl Buf, + boundary_sender: &mut impl Sender, + ) -> bool { + if self.artifact.is_some() || self.subscribers.is_empty() { + return false; + } + + let certificate = match Certificate::::decode_cfg( + message, + &self.verifier.certificate_codec_config(), + ) { + Ok(certificate) => certificate, + Err(err) => { + commonware_p2p::block!(self.blocker, peer, ?err, "invalid bootstrap certificate"); + return false; + } + }; + let Certificate::Finalization(finalization) = certificate else { + return false; + }; + if self + .pending + .as_ref() + .is_some_and(|pending| finalization.epoch() <= pending.epoch) + { + return false; + } + if !finalization.verify( + self.context.as_present_mut(), + &self.verifier, + &self.strategy, + ) { + commonware_p2p::block!(self.blocker, peer, "invalid bootstrap finalization"); + return false; + } + if finalization.epoch().is_zero() { + // Genesis needs no boundary block, but a strictly-newer finalization + // must still be able to supersede a replayed epoch-zero candidate. + // Record it as a pending candidate and resolve from local genesis only + // once the retry window elapses with nothing newer, matching the + // supersede window every non-zero epoch already gets. + self.pending = Some(Pending { + height: Height::zero(), + epoch: Epoch::zero(), + }); + return true; + } + + let Some(height) = finalization + .epoch() + .previous() + .and_then(|e| self.epocher.last(e)) + else { + warn!( + epoch = %finalization.epoch(), + "bootstrap finalization epoch has no boundary height" + ); + return false; + }; + Self::request_boundary(finalization.epoch(), boundary_sender); + self.pending = Some(Pending { + height, + epoch: finalization.epoch(), + }); + true + } + + /// Broadcast a request for the boundary block of `epoch` to all peers. + fn request_boundary(epoch: Epoch, boundary_sender: &mut impl Sender) { + boundary_sender.send( + Recipients::All, + wire::Message::::Request(epoch).encode(), + false, + ); + } + + fn handle_boundary_response(&mut self, peer: S::PublicKey, message: impl Buf) { + let Some(pending) = self.pending.take() else { + return; + }; + + let response = match wire::read_response_finalization::( + message, + &self.verifier.certificate_codec_config(), + ) { + Ok(Some(response)) => response, + Ok(None) => { + self.pending = Some(pending); + return; + } + Err(err) => { + commonware_p2p::block!( + self.blocker, + peer, + ?err, + "invalid bootstrap boundary response" + ); + self.pending = Some(pending); + return; + } + }; + let finalization = response.finalization; + let body = response.body; + + let Some(expected_finalization_epoch) = pending.epoch.previous() else { + commonware_p2p::block!(self.blocker, peer, "invalid bootstrap boundary response"); + self.pending = Some(pending); + return; + }; + + let response_finalization_epoch = finalization.epoch(); + if response_finalization_epoch < expected_finalization_epoch { + debug!( + response_finalization_epoch = %response_finalization_epoch, + pending_epoch = %pending.epoch, + "ignoring stale bootstrap boundary response" + ); + self.pending = Some(pending); + return; + } + + if response_finalization_epoch != expected_finalization_epoch { + commonware_p2p::block!(self.blocker, peer, "invalid bootstrap boundary response"); + self.pending = Some(pending); + return; + } + + let response = match authenticate_boundary_response::<_, S, V, _>( + self.context.as_present_mut(), + &self.verifier, + &self.strategy, + &self.block_codec_config, + finalization, + body, + ) { + Ok(response) => response, + Err(BoundaryResponseError::Decode(err)) => { + commonware_p2p::block!( + self.blocker, + peer, + ?err, + "invalid bootstrap boundary response" + ); + self.pending = Some(pending); + return; + } + Err( + BoundaryResponseError::BlockCommitment | BoundaryResponseError::InvalidFinalization, + ) => { + commonware_p2p::block!(self.blocker, peer, "invalid bootstrap boundary response"); + self.pending = Some(pending); + return; + } + }; + + match self.artifact_from_response(&pending, response) { + Some(artifact) => self.resolve(artifact), + None => { + commonware_p2p::block!(self.blocker, peer, "invalid bootstrap boundary response"); + self.pending = Some(pending); + } + } + } + + fn artifact_from_response( + &mut self, + pending: &Pending, + response: wire::Response, + ) -> Option> { + if response.block.height() != pending.height { + return None; + } + + let block = V::into_inner(response.block); + let Some(Payload::EpochInfo(info)) = block.payload() else { + return None; + }; + if info.epoch != pending.epoch { + return None; + } + + Some(Artifact { + epoch: info.epoch, + finalization: Some(response.finalization), + info, + }) + } + + fn resolve(&mut self, artifact: ActorArtifact) { + self.subscribers.drain(..).for_each(|subscriber| { + subscriber.send_lossy(artifact.clone()); + }); + self.artifact = Some(artifact); + } +} + +fn authenticate_boundary_response( + rng: &mut R, + verifier: &S, + strategy: &T, + block_codec_config: &::Cfg, + finalization: commonware_consensus::simplex::types::Finalization, + body: impl Buf, +) -> Result, BoundaryResponseError> +where + R: CryptoRng, + S: Scheme, + V: Variant, + T: Strategy, +{ + if !finalization.verify(rng, verifier, strategy) { + return Err(BoundaryResponseError::InvalidFinalization); + } + + let response = wire::read_response_block(body, finalization, block_codec_config) + .map_err(BoundaryResponseError::Decode)?; + if V::commitment(&response.block) != response.finalization.proposal.payload { + return Err(BoundaryResponseError::BlockCommitment); + } + + Ok(response) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dkg::tests::mocks; + use commonware_codec::Write as _; + use commonware_coding::ReedSolomon; + use commonware_consensus::{ + CertifiableBlock, + marshal::coding::{ + Coding, + types::{CodedBlock, coding_config_for_participants}, + }, + simplex::{ + scheme::bls12381_threshold::vrf::Scheme as ThresholdScheme, + types::{Finalization, Finalize, Proposal}, + }, + types::{Epoch, Height, Round, View, coding::Commitment}, + }; + use commonware_cryptography::{ + Digest as _, Digestible as _, Hasher as _, bls12381::primitives::variant::MinPk, + sha256::Sha256, + }; + use commonware_parallel::Sequential; + use commonware_runtime::{Runner as _, deterministic}; + use std::time::Duration; + + const THRESHOLD_NAMESPACE: &[u8] = b"_COMMONWARE_GLUE_DKG_ANCHOR_DISCOVERY_TEST"; + + type CodingContext = + commonware_consensus::simplex::types::Context; + type CodingBlock = mocks::MockBlock; + type TestCodingVariant = Coding, Sha256, mocks::TestPublicKey>; + type TestThresholdScheme = ThresholdScheme; + + impl CertifiableBlock for CodingBlock { + type Context = CodingContext; + + fn context(&self) -> Self::Context { + self.context().clone() + } + } + + fn finalization(proposal: Proposal, schemes: &[S]) -> Finalization + where + D: commonware_cryptography::Digest, + S: commonware_consensus::simplex::scheme::Scheme, + { + let finalizes = schemes + .iter() + .map(|scheme| Finalize::sign(scheme, proposal.clone()).unwrap()) + .collect::>(); + Finalization::from_finalizes(&schemes[0], &finalizes, &Sequential) + .expect("finalization quorum") + } + + fn split_response<'a, S, V>( + message: &'a [u8], + verifier: &S, + ) -> (Finalization, &'a [u8]) + where + S: Scheme, + V: Variant, + { + let response = wire::read_response_finalization::( + message, + &verifier.certificate_codec_config(), + ) + .expect("response decoded") + .expect("response tag"); + (response.finalization, response.body) + } + + fn threshold_fixture( + context: &mut deterministic::Context, + ) -> commonware_cryptography::certificate::mocks::Fixture { + commonware_consensus::simplex::scheme::bls12381_threshold::vrf::fixture::( + context, + THRESHOLD_NAMESPACE, + 4, + ) + } + + fn coding_block( + leader: mocks::TestPublicKey, + participants: u16, + ) -> CodedBlock, Sha256> { + let parent = Sha256::hash(b"parent"); + let context = CodingContext { + round: Round::new(Epoch::zero(), View::new(1)), + leader, + parent: ( + View::zero(), + Commitment::from(( + mocks::TestDigest::EMPTY, + mocks::TestDigest::EMPTY, + mocks::TestDigest::EMPTY, + coding_config_for_participants(participants), + )), + ), + }; + let block = CodingBlock::new::(context, parent, Height::new(1), 0); + CodedBlock::new( + block, + coding_config_for_participants(participants), + &Sequential, + ) + } + + #[test] + fn invalid_coding_finalization_is_rejected_before_block_decode() { + let runner = deterministic::Runner::timed(Duration::from_secs(5)); + runner.start(|mut context| async move { + let fixture = threshold_fixture(&mut context); + let verifier = TestThresholdScheme::certificate_verifier( + THRESHOLD_NAMESPACE, + *fixture.verifier.identity(), + ); + let block = coding_block( + fixture.participants[0].clone(), + fixture + .participants + .len() + .try_into() + .expect("participant count fits u16"), + ); + let payload = TestCodingVariant::commitment(&block); + let mut finalization = finalization( + Proposal::new( + Round::new(Epoch::zero(), View::new(1)), + View::zero(), + payload, + ), + &fixture.schemes, + ); + finalization.proposal.payload = Commitment::from(( + Sha256::hash(b"tampered block"), + Sha256::hash(b"tampered root"), + Sha256::hash(b"tampered context"), + coding_config_for_participants( + fixture + .participants + .len() + .try_into() + .expect("participant count fits u16"), + ), + )); + let mut message = Vec::new(); + wire::Tag::Response.write(&mut message); + finalization.write(&mut message); + message.extend_from_slice(b"body must not be decoded"); + + let (finalization, body) = + split_response::(&message, &verifier); + let result = + authenticate_boundary_response::<_, TestThresholdScheme, TestCodingVariant, _>( + &mut context, + &verifier, + &Sequential, + &(), + finalization, + body, + ); + + assert!(matches!( + result, + Err(BoundaryResponseError::InvalidFinalization) + )); + }); + } + + #[test] + fn valid_standard_response_decodes_after_authentication() { + let runner = deterministic::Runner::timed(Duration::from_secs(5)); + runner.start(|mut context| async move { + let fixture = mocks::scheme_fixture_n(&mut context, 4); + let block = mocks::genesis_block(fixture.participants[0].clone()); + let finalization = finalization( + Proposal::new( + Round::new(Epoch::zero(), View::new(1)), + View::zero(), + block.digest(), + ), + &fixture.schemes, + ); + let message = wire::Message::::Response( + wire::Response { + finalization, + block: block.clone(), + }, + ) + .encode() + .to_vec(); + let (finalization, body) = + split_response::( + &message, + &fixture.schemes[0], + ); + + let response = authenticate_boundary_response::< + _, + mocks::TestScheme, + mocks::TestMarshalVariant, + _, + >( + &mut context, + &fixture.schemes[0], + &Sequential, + &(), + finalization, + body, + ) + .expect("standard response authenticated"); + + assert_eq!(response.block, block); + }); + } + + #[test] + fn valid_coding_response_decodes_after_authentication() { + let runner = deterministic::Runner::timed(Duration::from_secs(5)); + runner.start(|mut context| async move { + let fixture = mocks::scheme_fixture_n(&mut context, 4); + let block = coding_block( + fixture.participants[0].clone(), + fixture + .participants + .len() + .try_into() + .expect("participant count fits u16"), + ); + let payload = TestCodingVariant::commitment(&block); + let finalization = finalization( + Proposal::new( + Round::new(Epoch::zero(), View::new(1)), + View::zero(), + payload, + ), + &fixture.schemes, + ); + let message = + wire::Message::::Response(wire::Response { + finalization, + block, + }) + .encode() + .to_vec(); + let (finalization, body) = split_response::( + &message, + &fixture.schemes[0], + ); + + let response = + authenticate_boundary_response::<_, mocks::TestScheme, TestCodingVariant, _>( + &mut context, + &fixture.schemes[0], + &Sequential, + &(), + finalization, + body, + ) + .expect("coding response authenticated"); + + assert_eq!(response.block.height(), Height::new(1)); + assert_eq!(TestCodingVariant::commitment(&response.block), payload); + }); + } + + #[test] + fn valid_coding_response_decodes_with_certificate_verifier() { + let runner = deterministic::Runner::timed(Duration::from_secs(5)); + runner.start(|mut context| async move { + let fixture = threshold_fixture(&mut context); + let verifier = TestThresholdScheme::certificate_verifier( + THRESHOLD_NAMESPACE, + *fixture.verifier.identity(), + ); + let block = coding_block( + fixture.participants[0].clone(), + fixture + .participants + .len() + .try_into() + .expect("participant count fits u16"), + ); + let payload = TestCodingVariant::commitment(&block); + let finalization = finalization( + Proposal::new( + Round::new(Epoch::zero(), View::new(1)), + View::zero(), + payload, + ), + &fixture.schemes, + ); + let message = + wire::Message::::Response(wire::Response { + finalization, + block, + }) + .encode() + .to_vec(); + let (finalization, body) = + split_response::(&message, &verifier); + + let response = + authenticate_boundary_response::<_, TestThresholdScheme, TestCodingVariant, _>( + &mut context, + &verifier, + &Sequential, + &(), + finalization, + body, + ) + .expect("coding response authenticated"); + + assert_eq!(response.block.height(), Height::new(1)); + assert_eq!(TestCodingVariant::commitment(&response.block), payload); + }); + } +} diff --git a/glue/src/dkg/anchor/actor/mod.rs b/glue/src/dkg/anchor/actor/mod.rs new file mode 100644 index 00000000000..feca0e24895 --- /dev/null +++ b/glue/src/dkg/anchor/actor/mod.rs @@ -0,0 +1,163 @@ +use super::mailbox::{Mailbox, Message}; +use crate::dkg::{ReshareBlock, types::EpochInfo}; +use commonware_actor::mailbox::{self as actor_mailbox, Receiver as ActorReceiver}; +use commonware_codec::Read; +use commonware_consensus::{marshal::core::Variant, simplex::scheme::Scheme, types::FixedEpocher}; +use commonware_cryptography::Signer; +use commonware_p2p::{Blocker, Channel, Manager, Message as P2pMessage, Receiver, Sender}; +use commonware_parallel::Strategy; +use commonware_runtime::{Clock, ContextCell, Handle, Metrics, Spawner, spawn_cell}; +use commonware_utils::{NonZeroDuration, channel::mpsc, ordered::Set}; +use discovery::Discovery; +use rand_core::CryptoRng; +use std::num::{NonZeroU64, NonZeroUsize}; + +mod discovery; +mod serving; + +/// Peer-set slot used before epoch-scoped tracking can supersede bootstrap peers. +const BOOTSTRAP_PEER_SET_INDEX: u64 = 0; + +/// Configuration for the anchor actor. +pub struct Config +where + E: Spawner + CryptoRng + Clock + Metrics, + M: Manager, + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, + ::Signer: Signer, + T: Strategy, + B: Blocker, +{ + /// Runtime context. + pub context: E, + /// P2P manager used to track the configured bootstrap peers. + pub manager: M, + /// Bootstrap peers used before epoch-scoped participants are known. + pub peers: Set, + /// All-epoch certificate verifier built from the constant BLS identity. + pub verifier: S, + /// Public epoch information carried by genesis. + pub genesis: EpochInfo<::Variant, S::PublicKey>, + /// Strategy for certificate verification. + pub strategy: T, + /// Blocker used to block peers that send invalid bootstrap data. + pub blocker: B, + /// Number of blocks in each epoch. + pub blocks_per_epoch: NonZeroU64, + /// How long to wait for a boundary response before re-broadcasting the request. + pub retry_timeout: NonZeroDuration, + /// Mailbox capacity. + pub mailbox_size: NonZeroUsize, + /// Codec configuration for application blocks received in boundary responses. + pub block_codec_config: ::Cfg, +} + +/// Anchor actor. +pub struct Actor +where + E: Spawner + CryptoRng + Clock + Metrics, + M: Manager, + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, + ::Signer: Signer, + T: Strategy, + B: Blocker, +{ + context: ContextCell, + mailbox: ActorReceiver>, + manager: M, + peers: Set, + verifier: S, + genesis: EpochInfo<::Variant, S::PublicKey>, + strategy: T, + blocker: B, + blocks_per_epoch: NonZeroU64, + retry_timeout: NonZeroDuration, + block_codec_config: ::Cfg, +} + +impl Actor +where + E: Spawner + CryptoRng + Clock + Metrics, + M: Manager, + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, + ::Signer: Signer, + T: Strategy, + B: Blocker, +{ + /// Create a anchor actor and mailbox. + pub fn new(config: Config) -> (Self, Mailbox) { + let (sender, mailbox) = + actor_mailbox::new(config.context.child("mailbox"), config.mailbox_size); + let mailbox_handle = Mailbox::new(sender); + ( + Self { + context: ContextCell::new(config.context), + mailbox, + manager: config.manager, + peers: config.peers, + verifier: config.verifier, + genesis: config.genesis, + strategy: config.strategy, + blocker: config.blocker, + blocks_per_epoch: config.blocks_per_epoch, + retry_timeout: config.retry_timeout, + block_codec_config: config.block_codec_config, + }, + mailbox_handle, + ) + } + + /// Start the anchor actor. + /// + /// The certificate backup channel is the mux backup receiver for the + /// physical Simplex certificate channel. The boundary network is the + /// anchor request channel used to fetch and later serve finalized boundary + /// blocks. + pub fn start( + mut self, + certificates: mpsc::Receiver<(Channel, P2pMessage)>, + boundaries: (BSE, BRE), + ) -> Handle<()> + where + BSE: Sender, + BRE: Receiver, + { + spawn_cell!(self.context, self.run(certificates, boundaries,)) + } + + async fn run( + mut self, + certificate_receiver: mpsc::Receiver<(Channel, P2pMessage)>, + (boundary_sender, boundary_receiver): (BSE, BRE), + ) where + BSE: Sender, + BRE: Receiver, + { + let _ = self + .manager + .track(BOOTSTRAP_PEER_SET_INDEX, self.peers.clone()); + + Discovery { + context: self.context, + mailbox: self.mailbox, + verifier: self.verifier, + genesis: self.genesis, + strategy: self.strategy, + blocker: self.blocker, + epocher: FixedEpocher::new(self.blocks_per_epoch), + block_codec_config: self.block_codec_config, + retry_timeout: self.retry_timeout, + artifact: None, + subscribers: Vec::new(), + pending: None, + } + .run(certificate_receiver, boundary_sender, boundary_receiver) + .await; + } +} diff --git a/glue/src/dkg/anchor/actor/serving.rs b/glue/src/dkg/anchor/actor/serving.rs new file mode 100644 index 00000000000..00e8c356ddf --- /dev/null +++ b/glue/src/dkg/anchor/actor/serving.rs @@ -0,0 +1,123 @@ +use crate::dkg::{ + ReshareBlock, + anchor::{ActorArtifact, mailbox::Message, wire}, +}; +use commonware_actor::mailbox::Receiver as ActorReceiver; +use commonware_codec::Encode as _; +use commonware_consensus::{ + marshal::core::{Mailbox as MarshalMailbox, Variant}, + simplex::scheme::Scheme, + types::{Epoch, Epocher, FixedEpocher}, +}; +use commonware_macros::select_loop; +use commonware_p2p::{Blocker, Receiver, Recipients, Sender}; +use commonware_runtime::{Clock, ContextCell, Metrics, Spawner}; +use commonware_utils::channel::fallible::OneshotExt as _; +use futures::{ + future::{self, Either}, + join, +}; +use rand_core::CryptoRng; +use tracing::debug; + +/// The boundary-serving phase of the anchor actor. +/// +/// Answers peers' boundary requests from the attached marshal. By construction +/// it does not listen to the Simplex certificate channel or issue outbound +/// discovery requests. +pub(super) struct Serving +where + E: Spawner + CryptoRng + Clock + Metrics, + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, + B: Blocker, +{ + pub(super) context: ContextCell, + pub(super) mailbox: ActorReceiver>, + pub(super) marshal: MarshalMailbox, + pub(super) blocker: B, + pub(super) epocher: FixedEpocher, + pub(super) artifact: Option>, +} + +impl Serving +where + E: Spawner + CryptoRng + Clock + Metrics, + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, + B: Blocker, +{ + /// Runs the serving loop until the actor shuts down. + pub(super) async fn run( + mut self, + mut sender: impl Sender, + mut receiver: impl Receiver, + ) { + let mut mailbox_drained = false; + select_loop! { + self.context, + on_start => { + let mailbox_message = if mailbox_drained { + Either::Left(future::pending()) + } else { + Either::Right(self.mailbox.recv()) + }; + }, + on_stopped => { + debug!("shutdown signal received"); + return; + }, + Some(message) = mailbox_message else { + mailbox_drained = true; + continue; + } => match message { + Message::Subscribe { response } => { + if let Some(artifact) = &self.artifact { + response.send_lossy(artifact.clone()); + } + } + Message::Attach { .. } => {} + }, + Ok((peer, message)) = receiver.recv() else { + debug!("boundary receiver closed, shutting down"); + return; + } => { + let epoch = match wire::read_request(message) { + Ok(Some(epoch)) => epoch, + Ok(None) => continue, + Err(err) => { + commonware_p2p::block!( + self.blocker, + peer, + ?err, + "invalid bootstrap boundary request" + ); + continue; + } + }; + let Some(response) = self.produce(epoch).await else { + continue; + }; + sender.send( + Recipients::One(peer), + wire::Message::::Response(response).encode(), + false, + ); + }, + } + } + + async fn produce(&mut self, epoch: Epoch) -> Option> { + let height = self.epocher.last(epoch.previous()?)?; + let (finalization, block) = join!( + self.marshal.get_finalization(height), + self.marshal.get_block(height) + ); + Some(wire::Response { + finalization: finalization?, + block: block?, + }) + } +} diff --git a/glue/src/dkg/anchor/mailbox.rs b/glue/src/dkg/anchor/mailbox.rs new file mode 100644 index 00000000000..95a2ffe5062 --- /dev/null +++ b/glue/src/dkg/anchor/mailbox.rs @@ -0,0 +1,86 @@ +use super::ActorArtifact; +use crate::dkg::ReshareBlock; +use commonware_actor::mailbox::{Policy, Sender}; +use commonware_consensus::{ + marshal::core::{Mailbox as MarshalMailbox, Variant}, + simplex::scheme::Scheme, +}; +use commonware_utils::channel::oneshot; +use std::collections::VecDeque; + +/// Messages sent to the anchor actor. +pub(crate) enum Message +where + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, +{ + /// Subscribe to the anchor artifact. + Subscribe { + /// Channel used to resolve the subscriber. + response: oneshot::Sender>, + }, + /// Attach marshal and transition to boundary-serving mode once discovery no + /// no longer has pending subscribers. + Attach { + /// Marshal mailbox used to serve boundary requests. + marshal: MarshalMailbox, + }, +} + +impl Policy for Message +where + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, +{ + type Overflow = VecDeque; + + fn handle(overflow: &mut Self::Overflow, message: Self) { + overflow.push_back(message); + } +} + +/// Mailbox for a running anchor actor. +#[derive(Clone)] +pub struct Mailbox +where + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, +{ + sender: Sender>, +} + +impl Mailbox +where + S: Scheme, + V: Variant, + V::ApplicationBlock: ReshareBlock, +{ + pub(crate) const fn new(sender: Sender>) -> Self { + Self { sender } + } + + /// Subscribe to the anchor artifact. + /// + /// The first live subscriber causes discovery to pay attention to the + /// Simplex certificate channel. Dropping the returned receiver cancels the + /// subscription. If discovery has already resolved, late subscribers receive + /// the cached artifact immediately. + pub fn subscribe(&self) -> oneshot::Receiver> { + let (response, receiver) = oneshot::channel(); + let _ = self.sender.enqueue(Message::Subscribe { response }); + receiver + } + + /// Attach marshal so the actor can serve peers' boundary requests. + /// + /// If discovery has pending subscribers, the actor waits until they are + /// resolved or dropped before entering serving. A source node can attach + /// marshal without ever subscribing, causing it to serve boundaries without + /// issuing discovery requests. + pub fn attach(&self, marshal: MarshalMailbox) { + let _ = self.sender.enqueue(Message::Attach { marshal }); + } +} diff --git a/glue/src/dkg/anchor/mod.rs b/glue/src/dkg/anchor/mod.rs new file mode 100644 index 00000000000..9cf3b63574e --- /dev/null +++ b/glue/src/dkg/anchor/mod.rs @@ -0,0 +1,1004 @@ +//! Discover the public epoch material a joining node needs before consensus starts. +//! +//! A node that is starting fresh cannot construct epoch-scoped state until it learns the current +//! epoch's participant set. That set lives in the [`EpochInfo`] of +//! a finalized boundary block. The [`Actor`] discovers that block, publishes the resulting +//! [`Artifact`], and then serves the same boundary block to other joining peers. +//! +//! At startup the node knows only a configured peer set, a constant certificate verifier valid +//! across all epochs, and the epoch length. +//! +//! # Weak Subjectivity +//! +//! The actor has a deliberately weak subjectivity boundary. Before it has read a boundary block it +//! does not know the epoch participant set, so it cannot run the `f + 1` peer-sample protocol used +//! by a stronger scheme such as [`stateful::probe`](crate::stateful::probe). Instead it accepts a +//! finalization it can verify with the all-epoch verifier, then asks peers for the boundary block +//! that finalization implies. A verified finalization remains provisional until the corresponding +//! boundary block is returned; while waiting, a strictly newer verified finalization may replace the +//! pending candidate. While the resulting [`Artifact`] is guaranteed to be a _valid_ finalized +//! block, it is not guaranteed that it is _recent_. +//! +//! Operators treat the configured peers and constant verifier as the weakly subjective checkpoint +//! for startup. +//! +//! # Protocol +//! +//! The actor is a two-state machine: it discovers an [`Artifact`], then serves boundary blocks. +//! +//! ## Discovery +//! +//! Once a subscriber appears, [`Actor`] listens on the simplex certificate channel and accepts a +//! finalization it can verify with the all-epoch verifier: +//! +//! ```text +//! peer --Finalization--> Actor --verify (all-epoch)--> accepted +//! ``` +//! +//! The accepted finalization names a finalized block, but not its contents. The actor asks peers +//! for the boundary block that holds the epoch's [`EpochInfo`]: +//! +//! ```text +//! +-- Request(epoch) --> peer 1 +//! | +//! Actor -------+-- Request(epoch) --> peer 2 <-- Response(block + finalization) +//! | +//! +-- Request(epoch) --> peer 3 +//! ``` +//! +//! If the boundary request remains unanswered and the actor verifies a newer finalization, that +//! finalization supersedes the pending candidate. The actor then requests the boundary implied by +//! the newer finalization: the final block of the previous epoch, which carries the newer epoch's +//! [`EpochInfo`]. Older or equal finalizations are ignored while a request is pending, and late +//! responses for superseded candidates are ignored without blocking the peer. +//! +//! ```text +//! peer --Finalization(epoch = E + 1)--> Actor --verify (all-epoch)--> supersede +//! Actor --Request(E + 1)--> peers +//! ``` +//! +//! The block's [`EpochInfo`] is packaged into an [`Artifact`] and +//! published to subscribers: +//! +//! ```text +//! finalization + boundary block --> Artifact { epoch, finalization, info } +//! ``` +//! +//! ## Serving +//! +//! After a source of finalized blocks is attached, the actor enters service and answers peers' +//! boundary requests for the rest of the process lifetime: +//! +//! ```text +//! peer --Request(epoch)--> Actor --lookup--> Response(block + finalization) --> peer +//! ``` +//! +//! An epoch with no known boundary block is answered with nothing. + +use crate::dkg::{ReshareBlock, types::EpochInfo}; +use commonware_consensus::{ + marshal::core::Variant as MarshalVariant, + simplex::{scheme::Scheme, types::Finalization}, + types::Epoch, +}; +use commonware_cryptography::{Digest, bls12381::primitives::variant::Variant as BlsVariant}; + +mod actor; +pub use actor::{Actor, Config}; + +mod mailbox; +pub use mailbox::Mailbox; + +mod wire; + +/// Concrete anchor artifact for a marshal variant. +pub(crate) type ActorArtifact = Artifact< + S, + ::Commitment, + <::ApplicationBlock as ReshareBlock>::Variant, +>; + +/// Public epoch material discovered during bootstrap. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Artifact +where + S: Scheme, + D: Digest, + V: BlsVariant, +{ + /// Epoch described by the boundary block's [`EpochInfo`]. + pub epoch: Epoch, + /// Finalization of the boundary block that carried the epoch info. + /// + /// Epoch zero is anchored by genesis and has no boundary finalization. + pub finalization: Option>, + /// Public epoch information from the finalized boundary block. + pub info: EpochInfo, +} + +#[cfg(test)] +mod tests { + use super::{Actor, Config, wire}; + use crate::dkg::{ + anchor::Artifact, + tests::mocks, + types::{EpochInfo, EpochOutcome, Payload}, + }; + use commonware_actor::Feedback; + use commonware_codec::Encode as _; + use commonware_consensus::{ + Epochable as _, Heightable as _, Reporter as _, + marshal::{self, Start, resolver::p2p as marshal_resolver}, + simplex::types::{Activity, Certificate, Finalization, Finalize, Proposal}, + types::{Epoch, Epocher as _, FixedEpocher, Height, Round, View, ViewDelta}, + }; + use commonware_cryptography::{ + Digest as _, Digestible as _, Hasher as _, + bls12381::{ + dkg::feldman_desmedt::deal, + primitives::sharing::{Mode, Sharing}, + }, + certificate::Verifier as _, + sha256::Sha256, + }; + use commonware_macros::select; + use commonware_p2p::{ + Receiver as _, Recipients, Sender as _, + simulated::{ + Config as NetworkConfig, Link, Network, Oracle, Receiver as SimReceiver, + Sender as SimSender, + }, + }; + use commonware_parallel::Sequential; + use commonware_runtime::{ + Clock as _, Handle, Quota, Runner as _, Spawner as _, Supervisor as _, + buffer::paged::CacheRef, deterministic, + }; + use commonware_storage::archive::immutable; + use commonware_utils::{ + N3f1, NZDuration, NZU16, NZU32, NZU64, NZUsize, TestRng, + channel::{fallible::AsyncFallibleExt as _, mpsc, oneshot}, + ordered::Set, + }; + use std::{num::NonZeroU64, time::Duration}; + + const BACKFILL_CHANNEL: u64 = 0; + const CERTIFICATE_CHANNEL: u64 = 1; + const BOUNDARY_CHANNEL: u64 = 2; + const TEST_QUOTA: Quota = Quota::per_second(NZU32!(1_000_000)); + const BLOCKS_PER_EPOCH: NonZeroU64 = NZU64!(2); + const LINK: Link = Link { + latency: Duration::from_millis(1), + jitter: Duration::ZERO, + success_rate: 1.0, + }; + + struct Harness { + participants: Vec, + schemes: Vec, + source_certificate_sender: SimSender, + source_boundary_sender: SimSender, + client_boundary_sender: SimSender, + client_boundary_receiver: SimReceiver, + oracle: Oracle, + joiner: super::Mailbox, + boundary: mocks::TestBlock, + boundary_finalization: Finalization, + boundary_sharing: Sharing, + _handles: Vec>, + _network: Handle<()>, + } + + impl Harness { + async fn start(context: &mut deterministic::Context) -> Self { + Self::start_with(context, true).await + } + + async fn start_with(context: &mut deterministic::Context, source_serves: bool) -> Self { + let boundaries = if source_serves { + vec![Epoch::new(1)] + } else { + Vec::new() + }; + Self::start_with_boundaries(context, boundaries).await + } + + async fn start_with_boundaries( + context: &mut deterministic::Context, + source_boundaries: Vec, + ) -> Self { + let fixture = mocks::scheme_fixture_n(context, 4); + let participants = fixture.participants.clone(); + let peers = Set::from_iter_dedup(participants.iter().cloned()); + + let (network, oracle) = Network::new_with_peers( + context.child("network"), + NetworkConfig { + max_size: 1024 * 1024, + disconnect_on_block: true, + tracked_peer_sets: NZUsize!(1), + }, + participants.clone(), + ) + .await; + let network = network.start(); + for from in &participants { + for to in &participants { + if from != to { + oracle + .add_link(from.clone(), to.clone(), LINK) + .await + .expect("failed to add link"); + } + } + } + + let (boundary, boundary_sharing) = + boundary_block(Epoch::new(1), participants[0].clone(), &participants); + let genesis = genesis_info(&participants); + let first_boundary_finalization = + boundary_finalization(Epoch::new(1), boundary.digest(), &fixture.schemes); + let source_boundaries = source_boundaries + .into_iter() + .map(|epoch| { + let (block, _) = boundary_block(epoch, participants[0].clone(), &participants); + let finalization = + boundary_finalization(epoch, block.digest(), &fixture.schemes); + (block, finalization) + }) + .collect::>(); + + let (source_marshal, marshal_handle) = start_marshal( + context.child("source_marshal"), + &oracle, + &fixture, + 0, + source_boundaries, + ) + .await; + + let source_control = oracle.control(participants[0].clone()); + let (source_certificate_sender, _source_certificate_receiver) = source_control + .register(CERTIFICATE_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register source certificates"); + let (_source_certificate_backup_sender, source_certificate_backup) = mpsc::channel(16); + let source_boundaries = source_control + .register(BOUNDARY_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register source boundaries"); + let source_boundary_sender = source_boundaries.0.clone(); + let (source_actor, source_mailbox) = Actor::new(Config { + context: context.child("source_anchor"), + manager: oracle.manager(), + peers: peers.clone(), + verifier: fixture.schemes[0].clone(), + genesis: genesis.clone(), + strategy: Sequential, + blocker: oracle.control(participants[0].clone()), + blocks_per_epoch: BLOCKS_PER_EPOCH, + retry_timeout: NZDuration!(Duration::from_millis(500)), + mailbox_size: NZUsize!(16), + block_codec_config: (), + }); + source_mailbox.attach(source_marshal.clone()); + let source_handle = source_actor.start(source_certificate_backup, source_boundaries); + + let joiner_control = oracle.control(participants[1].clone()); + let (_joiner_certificate_sender, mut joiner_certificate_receiver) = joiner_control + .register(CERTIFICATE_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register joiner certificates"); + let (joiner_certificate_backup_sender, joiner_certificate_backup_receiver) = + mpsc::channel(16); + let joiner_certificate_backup_handle = context + .child("joiner_certificate_backup") + .spawn(move |_| async move { + loop { + let Ok(message) = joiner_certificate_receiver.recv().await else { + return; + }; + if !joiner_certificate_backup_sender + .send_lossy((CERTIFICATE_CHANNEL, message)) + .await + { + return; + } + } + }); + let joiner_boundaries = joiner_control + .register(BOUNDARY_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register joiner boundaries"); + let (joiner_actor, joiner) = Actor::new(Config { + context: context.child("joiner_anchor"), + manager: oracle.manager(), + peers, + verifier: fixture.schemes[1].clone(), + genesis, + strategy: Sequential, + blocker: oracle.control(participants[1].clone()), + blocks_per_epoch: BLOCKS_PER_EPOCH, + retry_timeout: NZDuration!(Duration::from_millis(500)), + mailbox_size: NZUsize!(16), + block_codec_config: (), + }); + let joiner_handle = + joiner_actor.start(joiner_certificate_backup_receiver, joiner_boundaries); + let client_boundaries = oracle + .control(participants[2].clone()) + .register(BOUNDARY_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register client boundaries"); + + Self { + participants, + schemes: fixture.schemes, + source_certificate_sender, + source_boundary_sender, + client_boundary_sender: client_boundaries.0, + client_boundary_receiver: client_boundaries.1, + oracle, + joiner, + boundary, + boundary_finalization: first_boundary_finalization, + boundary_sharing, + _handles: vec![ + marshal_handle, + source_handle, + joiner_certificate_backup_handle, + joiner_handle, + ], + _network: network, + } + } + + fn send_target_finalization( + &mut self, + ) -> Finalization { + self.send_target_finalization_for(Epoch::new(1), self.boundary.digest()) + } + + fn send_target_finalization_for( + &mut self, + epoch: Epoch, + digest: mocks::TestDigest, + ) -> Finalization { + let target = finalization( + Proposal::new(Round::new(epoch, View::new(2)), View::new(1), digest), + &self.schemes, + ); + self.source_certificate_sender.send( + Recipients::One(self.participants[1].clone()), + Certificate::Finalization(target.clone()).encode().to_vec(), + false, + ); + target + } + + /// Awaits the next boundary request the joiner broadcasts, as observed by + /// a peer, and returns the requested epoch. + async fn next_client_request(&mut self) -> Epoch { + let (_, message) = self + .client_boundary_receiver + .recv() + .await + .expect("boundary request"); + wire::read_request(message) + .expect("decode boundary request") + .expect("boundary request tag") + } + } + + async fn start_marshal( + context: deterministic::Context, + oracle: &Oracle, + fixture: &mocks::SchemeFixture, + index: usize, + boundaries: Vec<( + mocks::TestBlock, + Finalization, + )>, + ) -> (mocks::TestMarshalMailbox, Handle<()>) { + let public_key = fixture.participants[index].clone(); + let partition_prefix = format!("anchor-node-{index}"); + let page_cache = CacheRef::from_pooler(&context, NZU16!(1024), NZUsize!(16)); + let control = oracle.control(public_key.clone()); + let backfill = control + .register(BACKFILL_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register marshal backfill"); + let resolver = marshal_resolver::init( + context.child("marshal_resolver"), + marshal_resolver::Config { + public_key: public_key.clone(), + peer_provider: oracle.manager(), + blocker: oracle.control(public_key.clone()), + mailbox_size: NZUsize!(16), + initial: Duration::from_secs(1), + timeout: Duration::from_secs(2), + fetch_retry_timeout: Duration::from_millis(100), + priority_requests: false, + priority_responses: false, + }, + backfill, + ); + let finalizations_by_height = + immutable::Archive::init(context.child("finalizations_by_height"), { + let _: () = mocks::TestScheme::certificate_codec_config_unbounded(); + archive_config( + &partition_prefix, + "finalizations_by_height", + page_cache.clone(), + (), + ) + }) + .await + .expect("failed to initialize finalizations archive"); + let finalized_blocks = immutable::Archive::init( + context.child("finalized_blocks"), + archive_config( + &partition_prefix, + "finalized_blocks", + page_cache.clone(), + (), + ), + ) + .await + .expect("failed to initialize finalized blocks archive"); + + let (marshal_actor, mut marshal, _) = marshal::core::Actor::init( + context.child("marshal"), + finalizations_by_height, + finalized_blocks, + marshal::Config { + provider: mocks::TestProvider::new(fixture.schemes[index].clone()), + epocher: FixedEpocher::new(BLOCKS_PER_EPOCH), + start: Start::Genesis(mocks::genesis_block(public_key)), + partition_prefix, + mailbox_size: NZUsize!(16), + view_retention_timeout: ViewDelta::new(8), + prunable_items_per_section: NZU64!(10), + page_cache, + replay_buffer: NZUsize!(1024), + key_write_buffer: NZUsize!(1024), + value_write_buffer: NZUsize!(1024), + block_codec_config: (), + max_repair: NZUsize!(4), + max_pending_acks: NZUsize!(4), + strategy: Sequential, + }, + ) + .await; + let handle = marshal_actor.start_unbuffered(mocks::MarshalApplication::default(), resolver); + + for (block, finalization) in boundaries { + assert!(marshal.certified(block.context().round, block).await); + assert_eq!( + marshal.report(Activity::Finalization(finalization)), + Feedback::Ok + ); + } + + (marshal, handle) + } + + fn archive_config( + prefix: &str, + name: &str, + page_cache: CacheRef, + codec_config: C, + ) -> immutable::Config { + immutable::Config { + metadata_partition: format!("{prefix}-{name}-metadata"), + freezer_table_partition: format!("{prefix}-{name}-freezer-table"), + freezer_table_initial_size: 64, + freezer_table_resize_frequency: 10, + freezer_table_resize_chunk_size: 10, + freezer_key_partition: format!("{prefix}-{name}-freezer-key"), + freezer_key_page_cache: page_cache, + freezer_value_partition: format!("{prefix}-{name}-freezer-value"), + freezer_value_target_size: 1024, + freezer_value_compression: None, + ordinal_partition: format!("{prefix}-{name}-ordinal"), + items_per_section: NZU64!(10), + codec_config, + replay_buffer: NZUsize!(1024), + freezer_key_write_buffer: NZUsize!(1024), + freezer_value_write_buffer: NZUsize!(1024), + ordinal_write_buffer: NZUsize!(1024), + } + } + + fn boundary_block( + epoch: Epoch, + leader: mocks::TestPublicKey, + participants: &[mocks::TestPublicKey], + ) -> (mocks::TestBlock, Sharing) { + let height = FixedEpocher::new(BLOCKS_PER_EPOCH) + .last(epoch.previous().expect("boundary epoch must be non-zero")) + .expect("test epoch must be supported"); + let parent = if height == Height::zero() { + mocks::TestDigest::EMPTY + } else { + Sha256::hash( + &height + .previous() + .expect("non-genesis height") + .get() + .to_be_bytes(), + ) + }; + let context = mocks::TestContext { + round: Round::new( + epoch.previous().expect("boundary epoch must be non-zero"), + View::new(1), + ), + leader, + parent: (View::zero(), parent), + }; + let participants = Set::from_iter_dedup(participants.iter().cloned()); + let (output, _) = deal::( + TestRng::new(epoch.get()), + Mode::NonZeroCounter, + participants.clone(), + ) + .expect("failed to create test DKG output"); + let sharing = output.public().clone(); + let block = mocks::TestBlock::new::(context, parent, height, epoch.get()) + .with_payload::( + NZU32!(16), + Payload::EpochInfo(EpochInfo { + outcome: EpochOutcome::Success, + epoch, + output, + players: participants.clone(), + next_players: participants, + }), + ); + (block, sharing) + } + + fn boundary_finalization( + epoch: Epoch, + digest: mocks::TestDigest, + schemes: &[mocks::TestScheme], + ) -> Finalization { + finalization( + Proposal::new( + Round::new( + epoch.previous().expect("boundary epoch must be non-zero"), + View::new(1), + ), + View::zero(), + digest, + ), + schemes, + ) + } + + fn genesis_info( + participants: &[mocks::TestPublicKey], + ) -> EpochInfo { + let participants = Set::from_iter_dedup(participants.iter().cloned()); + let (output, _) = deal::( + TestRng::new(0), + Mode::NonZeroCounter, + participants.clone(), + ) + .expect("failed to create test DKG output"); + EpochInfo { + outcome: EpochOutcome::Success, + epoch: Epoch::zero(), + output, + players: participants.clone(), + next_players: participants, + } + } + + fn finalization( + proposal: Proposal, + schemes: &[mocks::TestScheme], + ) -> Finalization { + let finalizes = schemes + .iter() + .map(|scheme| Finalize::sign(scheme, proposal.clone()).unwrap()) + .collect::>(); + Finalization::from_finalizes(&schemes[0], &finalizes, &Sequential) + .expect("finalization quorum") + } + + fn assert_artifact( + artifact: Artifact, + expected_finalization: &Finalization, + expected_sharing: &Sharing, + participants: &[mocks::TestPublicKey], + ) { + let expected_epoch = expected_finalization.epoch().next(); + let participants = Set::from_iter_dedup(participants.iter().cloned()); + assert_eq!(artifact.epoch, expected_epoch); + assert_eq!(artifact.finalization.as_ref(), Some(expected_finalization)); + assert_eq!(artifact.info.epoch, expected_epoch); + assert_eq!(artifact.info.output.public(), expected_sharing); + assert_eq!(artifact.info.output.players(), &participants); + assert_eq!(artifact.info.players, participants); + } + + #[test] + fn discovers_artifact_from_first_finalization() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + let mut harness = Harness::start(&mut context).await; + let mut subscription = harness.joiner.subscribe(); + harness.send_target_finalization(); + + context.sleep(Duration::from_millis(100)).await; + let artifact = subscription.try_recv().expect("artifact resolved"); + assert_artifact( + artifact, + &harness.boundary_finalization, + &harness.boundary_sharing, + &harness.participants, + ); + }); + } + + #[test] + fn rebroadcasts_boundary_request_when_unanswered() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + // The source has no boundary block, so the joiner's request goes + // unanswered and it must re-request rather than wedging. + let mut harness = Harness::start_with(&mut context, false).await; + let mut subscription = harness.joiner.subscribe(); + harness.send_target_finalization(); + + // First broadcast: a peer observes the request, but nobody answers. + assert_eq!(harness.next_client_request().await, Epoch::new(1)); + assert!(matches!( + subscription.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + + // After the retry timeout the joiner re-broadcasts the same request. + assert_eq!(harness.next_client_request().await, Epoch::new(1)); + assert!(matches!( + subscription.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + }); + } + + #[test] + fn newer_finalization_supersedes_unanswered_boundary() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + // The source can serve the newer boundary but not the first one. + let mut harness = + Harness::start_with_boundaries(&mut context, vec![Epoch::new(2)]).await; + let mut subscription = harness.joiner.subscribe(); + + harness.send_target_finalization(); + assert_eq!(harness.next_client_request().await, Epoch::new(1)); + assert!(matches!( + subscription.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + + let (newer_boundary, newer_sharing) = boundary_block( + Epoch::new(2), + harness.participants[0].clone(), + &harness.participants, + ); + let newer_finalization = + harness.send_target_finalization_for(Epoch::new(2), newer_boundary.digest()); + + assert_eq!(harness.next_client_request().await, Epoch::new(2)); + context.sleep(Duration::from_millis(100)).await; + + let artifact = subscription.try_recv().expect("artifact resolved"); + assert_artifact( + artifact, + &boundary_finalization(Epoch::new(2), newer_boundary.digest(), &harness.schemes), + &newer_sharing, + &harness.participants, + ); + assert_eq!(newer_finalization.epoch(), Epoch::new(2)); + }); + } + + #[test] + fn genesis_resolves_after_retry_grace() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + let mut harness = Harness::start(&mut context).await; + let mut subscription = harness.joiner.subscribe(); + + // A lone epoch-zero finalization is only provisional: it must not + // resolve until the retry grace elapses, so a newer finalization has + // a window to supersede it. + harness.send_target_finalization_for(Epoch::zero(), mocks::TestDigest::EMPTY); + context.sleep(Duration::from_millis(100)).await; + assert!(matches!( + subscription.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + + // With nothing newer arriving, the grace elapses and genesis resolves + // from the locally known artifact. + context.sleep(Duration::from_secs(1)).await; + let artifact = subscription.try_recv().expect("genesis resolved"); + assert_eq!(artifact.epoch, Epoch::zero()); + assert!(artifact.finalization.is_none()); + assert_eq!(artifact.info, genesis_info(&harness.participants)); + }); + } + + #[test] + fn newer_finalization_supersedes_genesis() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + // The source can serve the epoch-one boundary. + let mut harness = Harness::start(&mut context).await; + let mut subscription = harness.joiner.subscribe(); + + // A replayed epoch-zero finalization arrives first, but must not + // permanently pin the joiner to genesis. + harness.send_target_finalization_for(Epoch::zero(), mocks::TestDigest::EMPTY); + context.sleep(Duration::from_millis(100)).await; + assert!(matches!( + subscription.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + + // A strictly-newer epoch-one finalization supersedes the genesis + // candidate and resolves to the epoch-one boundary artifact. + harness.send_target_finalization(); + context.sleep(Duration::from_millis(100)).await; + let artifact = subscription + .try_recv() + .expect("newer finalization resolved"); + assert_artifact( + artifact, + &harness.boundary_finalization, + &harness.boundary_sharing, + &harness.participants, + ); + }); + } + + #[test] + fn late_response_for_superseded_boundary_does_not_block_peer() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + let mut harness = Harness::start_with_boundaries(&mut context, Vec::new()).await; + let mut subscription = harness.joiner.subscribe(); + + harness.send_target_finalization(); + assert_eq!(harness.next_client_request().await, Epoch::new(1)); + + let (newer_boundary, _) = boundary_block( + Epoch::new(2), + harness.participants[0].clone(), + &harness.participants, + ); + harness.send_target_finalization_for(Epoch::new(2), newer_boundary.digest()); + assert_eq!(harness.next_client_request().await, Epoch::new(2)); + + harness.source_boundary_sender.send( + Recipients::One(harness.participants[1].clone()), + wire::Message::::Response( + wire::Response { + finalization: harness.boundary_finalization.clone(), + block: harness.boundary.clone(), + }, + ) + .encode() + .to_vec(), + false, + ); + context.sleep(Duration::from_millis(100)).await; + + let blocked = harness.oracle.blocked().await.unwrap(); + assert!( + !blocked.contains(&( + harness.participants[1].clone(), + harness.participants[0].clone() + )), + "late response for superseded boundary should not block source peer" + ); + assert!(matches!( + subscription.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + }); + } + + #[test] + fn terminal_epoch_boundary_response_does_not_panic() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + let mut harness = Harness::start_with_boundaries(&mut context, Vec::new()).await; + let mut subscription = harness.joiner.subscribe(); + + harness.send_target_finalization(); + assert_eq!(harness.next_client_request().await, Epoch::new(1)); + + let terminal_finalization = finalization( + Proposal::new( + Round::new(Epoch::new(u64::MAX), View::new(1)), + View::zero(), + harness.boundary.digest(), + ), + &harness.schemes, + ); + let message = wire::Message::::Response( + wire::Response { + finalization: terminal_finalization, + block: harness.boundary.clone(), + }, + ) + .encode() + .to_vec(); + let decoded = wire::read_response_finalization::< + mocks::TestScheme, + mocks::TestMarshalVariant, + _, + >( + message.as_slice(), + &harness.schemes[2].certificate_codec_config(), + ) + .expect("terminal response decoded") + .expect("terminal response tag"); + let decoded = + wire::read_response_block::( + decoded.body, + decoded.finalization, + &(), + ) + .expect("terminal response block decoded"); + assert_eq!(decoded.finalization.epoch(), Epoch::new(u64::MAX)); + + harness.source_boundary_sender.send( + Recipients::One(harness.participants[1].clone()), + message, + false, + ); + context.sleep(Duration::from_millis(100)).await; + + let blocked = harness.oracle.blocked().await.unwrap(); + assert!( + blocked.contains(&( + harness.participants[1].clone(), + harness.participants[0].clone() + )), + "terminal-epoch response should block source peer" + ); + assert!(matches!( + subscription.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + }); + } + + #[test] + fn ignores_certificates_until_subscribed() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + let mut harness = Harness::start(&mut context).await; + harness.send_target_finalization(); + context.sleep(Duration::from_millis(100)).await; + + let mut subscription = harness.joiner.subscribe(); + context.sleep(Duration::from_millis(100)).await; + assert!(matches!( + subscription.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + + harness.send_target_finalization(); + context.sleep(Duration::from_millis(100)).await; + let artifact = subscription.try_recv().expect("artifact resolved"); + assert_artifact( + artifact, + &harness.boundary_finalization, + &harness.boundary_sharing, + &harness.participants, + ); + }); + } + + #[test] + fn late_subscriber_receives_cached_artifact() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + let mut harness = Harness::start(&mut context).await; + let mut first = harness.joiner.subscribe(); + harness.send_target_finalization(); + + context.sleep(Duration::from_millis(100)).await; + let artifact = first.try_recv().expect("artifact resolved"); + assert_artifact( + artifact, + &harness.boundary_finalization, + &harness.boundary_sharing, + &harness.participants, + ); + + let mut second = harness.joiner.subscribe(); + context.sleep(Duration::from_millis(10)).await; + let artifact = second.try_recv().expect("cached artifact resolved"); + assert_artifact( + artifact, + &harness.boundary_finalization, + &harness.boundary_sharing, + &harness.participants, + ); + }); + } + + #[test] + fn serving_answers_boundary_requests_from_marshal() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + let mut harness = Harness::start(&mut context).await; + harness.client_boundary_sender.send( + Recipients::One(harness.participants[0].clone()), + wire::Message::::Request(Epoch::new( + 1, + )) + .encode() + .to_vec(), + false, + ); + + let (_peer, message) = harness + .client_boundary_receiver + .recv() + .await + .expect("boundary response delivered"); + let response = wire::read_response_finalization::< + mocks::TestScheme, + mocks::TestMarshalVariant, + _, + >(message, &harness.schemes[2].certificate_codec_config()) + .expect("boundary response decoded") + .expect("boundary response"); + let response = + wire::read_response_block::( + response.body, + response.finalization, + &(), + ) + .expect("boundary response block decoded"); + + assert_eq!(response.block.digest(), harness.boundary.digest()); + assert_eq!(response.block.height(), Height::new(1)); + assert_eq!(response.finalization, harness.boundary_finalization); + }); + } + + #[test] + fn serving_ignores_epoch_without_boundary() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|mut context| async move { + let mut harness = Harness::start(&mut context).await; + harness.client_boundary_sender.send( + Recipients::One(harness.participants[0].clone()), + wire::Message::::Request( + Epoch::zero(), + ) + .encode() + .to_vec(), + false, + ); + + select! { + _ = harness.client_boundary_receiver.recv() => { + panic!("boundary response delivered"); + }, + _ = context.sleep(Duration::from_millis(100)) => {}, + }; + }); + } +} diff --git a/glue/src/dkg/anchor/wire.rs b/glue/src/dkg/anchor/wire.rs new file mode 100644 index 00000000000..fab5bef4f63 --- /dev/null +++ b/glue/src/dkg/anchor/wire.rs @@ -0,0 +1,192 @@ +use bytes::{Buf, BufMut}; +use commonware_codec::{Decode, DecodeExt, EncodeSize, Error, FixedSize, Read, ReadExt, Write}; +use commonware_consensus::{ + marshal::core::Variant, + simplex::{scheme::Scheme, types::Finalization}, + types::Epoch, +}; + +/// First byte of a anchor boundary message. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub(crate) enum Tag { + /// Request the finalized boundary block and finalization for an epoch. + Request, + /// Response carrying the finalized boundary block and finalization. + Response, +} + +impl FixedSize for Tag { + const SIZE: usize = u8::SIZE; +} + +impl Write for Tag { + fn write(&self, writer: &mut impl BufMut) { + match self { + Self::Request => 0u8.write(writer), + Self::Response => 1u8.write(writer), + } + } +} + +impl Read for Tag { + type Cfg = (); + + fn read_cfg(reader: &mut impl Buf, _: &()) -> Result { + match u8::read(reader)? { + 0 => Ok(Self::Request), + 1 => Ok(Self::Response), + n => Err(Error::InvalidEnum(n)), + } + } +} + +/// Boundary response decoded from a peer. +pub(crate) struct Response +where + S: Scheme, + V: Variant, +{ + pub(crate) finalization: Finalization, + pub(crate) block: V::Block, +} + +/// Boundary response after decoding the tag and finalization. +pub(crate) struct ResponseHeader +where + S: Scheme, + V: Variant, +{ + pub(crate) finalization: Finalization, + pub(crate) body: R, +} + +/// Anchor boundary request/response. +pub(crate) enum Message +where + S: Scheme, + V: Variant, +{ + /// Request the finalized boundary block and finalization for `epoch`. + Request(Epoch), + /// Respond with the finalized boundary block and finalization. + Response(Response), +} + +impl Write for Message +where + S: Scheme, + V: Variant, +{ + fn write(&self, writer: &mut impl BufMut) { + match self { + Self::Request(epoch) => { + Tag::Request.write(writer); + epoch.write(writer); + } + Self::Response(response) => { + Tag::Response.write(writer); + response.finalization.write(writer); + response.block.write(writer); + } + } + } +} + +impl EncodeSize for Message +where + S: Scheme, + V: Variant, +{ + fn encode_size(&self) -> usize { + Tag::SIZE + + match self { + Self::Request(epoch) => epoch.encode_size(), + Self::Response(response) => { + response.finalization.encode_size() + response.block.encode_size() + } + } + } +} + +#[cfg(feature = "arbitrary")] +impl arbitrary::Arbitrary<'_> for Message +where + S: Scheme, + V: Variant, + S::Certificate: for<'a> arbitrary::Arbitrary<'a>, + V::Commitment: for<'a> arbitrary::Arbitrary<'a>, + V::Block: for<'a> arbitrary::Arbitrary<'a>, +{ + fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { + Ok(match Tag::arbitrary(u)? { + Tag::Request => Self::Request(Epoch::arbitrary(u)?), + Tag::Response => Self::Response(Response { + finalization: Finalization::arbitrary(u)?, + block: V::Block::arbitrary(u)?, + }), + }) + } +} + +/// Decode a boundary request. +pub(crate) fn read_request(mut reader: impl Buf) -> Result, Error> { + let tag = Tag::read(&mut reader)?; + if tag != Tag::Request { + return Ok(None); + } + Ok(Some(Epoch::decode(reader)?)) +} + +/// Decode the tag and finalization of a boundary response. +pub(crate) fn read_response_finalization( + mut reader: R, + certificate_cfg: &::Cfg, +) -> Result>, Error> +where + S: Scheme, + V: Variant, + R: Buf, +{ + let tag = Tag::read(&mut reader)?; + if tag != Tag::Response { + return Ok(None); + } + + let finalization = Finalization::read_cfg(&mut reader, certificate_cfg)?; + Ok(Some(ResponseHeader { + finalization, + body: reader, + })) +} + +/// Decode the block body of a boundary response using an authenticated finalization. +pub(crate) fn read_response_block( + reader: impl Buf, + finalization: Finalization, + block_codec_config: &::Cfg, +) -> Result, Error> +where + S: Scheme, + V: Variant, +{ + let block_cfg = V::block_cfg(block_codec_config, finalization.proposal.payload); + let block = V::Block::decode_cfg(reader, &block_cfg)?; + + Ok(Response { + finalization, + block, + }) +} + +#[cfg(all(test, feature = "arbitrary"))] +mod tests { + use super::{Message, Tag}; + use crate::dkg::tests::mocks; + use commonware_codec::conformance::CodecConformance; + + commonware_conformance::conformance_tests! { + CodecConformance, + CodecConformance>, + } +} diff --git a/glue/src/dkg/bootstrap/mod.rs b/glue/src/dkg/bootstrap/mod.rs new file mode 100644 index 00000000000..81cd1a978be --- /dev/null +++ b/glue/src/dkg/bootstrap/mod.rs @@ -0,0 +1,628 @@ +//! One-shot engine for generating an initial BLS threshold output. +//! +//! The engine runs an independent Ed25519 Simplex chain for one epoch and uses +//! the reshare actor's crate-private DKG mode to perform the ceremony. +//! The resulting [`EpochInfo`] can be used as the genesis threshold artifact for +//! a separate application that will later run continuous resharing. +//! +//! See [`reshare`] for the protocol flow that this engine reuses and for the +//! application contract of a continuously reshared chain. + +use crate::dkg::{ + ParticipantsProvider, Registrar, ReshareBlock, SecretStore, + fence::Fence, + reshare::{self, DkgConfig}, + types::{EpochInfo, Participants, Payload, SchemeInfo}, +}; +use commonware_broadcast::buffered; +use commonware_codec::{Encode, EncodeSize, Error as CodecError, Read, ReadExt as _, Write}; +use commonware_consensus::{ + Application, Block as ConsensusBlock, CertifiableBlock, Heightable, + marshal::{ + self, Start, ancestry::Ancestry, core::Actor as MarshalActor, + resolver::p2p as marshal_resolver, standard::Deferred, + }, + simplex::{self, Floor, config::ForwardingPolicy, elector::RoundRobin, types::Context}, + types::{Epoch, FixedEpocher, Height, Round, View, ViewDelta}, +}; +use commonware_cryptography::{ + BatchVerifier, Digest as _, Digestible, Hasher, PublicKey, Sha256, Signer as _, + bls12381::primitives::{sharing::Mode as SharingMode, variant::Variant}, + certificate::{ConstantProvider, Verifier as _}, + ed25519, + sha256::{self, Digest as Sha256Digest}, +}; +use commonware_macros::select; +use commonware_p2p::{Blocker, Manager, Receiver, Sender}; +use commonware_parallel::Strategy; +use commonware_runtime::{ + Buf, BufMut, BufferPooler, Clock, ContextCell, Handle, Metrics, Spawner, Storage, + buffer::paged::CacheRef, spawn_cell, +}; +use commonware_storage::{archive::prunable, translator::TwoCap}; +use commonware_utils::{ + NZU16, NZU32, NZU64, NZUsize, + channel::{fallible::OneshotExt, oneshot}, + ordered::Set, +}; +use rand_core::{CryptoRng, Rng}; +use std::{ + marker::PhantomData, + num::{NonZeroU16, NonZeroU32, NonZeroU64, NonZeroUsize}, + time::Duration, +}; + +const MAILBOX_SIZE: NonZeroUsize = NZUsize!(100); +const PAGE_SIZE: NonZeroU16 = NZU16!(1024); +const PAGE_CACHE_PAGES: NonZeroUsize = NZUsize!(16); +const IO_BUFFER_SIZE: NonZeroUsize = NZUsize!(2048); +const ARCHIVE_ITEMS_PER_SECTION: NonZeroU64 = NZU64!(10); + +type ConsensusScheme = simplex::scheme::ed25519::Scheme; + +/// Configuration for [`Engine`]. +pub struct Config { + /// Ed25519 signer used for the one-shot consensus chain and DKG protocol messages. + pub signer: ed25519::PrivateKey, + + /// P2P manager used for peer tracking. + pub manager: M, + + /// Blocker used for invalid peer behavior. + pub blocker: X, + + /// User-owned store for private DKG material. + pub secret_store: SS, + + /// Parallel verification strategy. + pub strategy: T, + + /// Application namespace for DKG transcript separation. + pub namespace: &'static [u8], + + /// Sharing mode used for the generated threshold output. + pub sharing_mode: SharingMode, + + /// Runtime-storage partition prefix. + pub partition_prefix: String, + + /// Participants in the DKG. + pub participants: Set, + + /// Length of the one-shot consensus epoch. + pub blocks_per_epoch: NonZeroU64, +} + +/// Completion produced when the one-shot DKG chain finalizes its final block. +pub struct Completion { + /// Final DKG artifact, if the ceremony succeeded. + pub info: Option>, +} + +/// Block type used by the one-shot DKG chain. +#[derive(Clone, PartialEq, Eq)] +pub struct Block { + context: Context, + parent: sha256::Digest, + height: Height, + payload: Option>, +} + +impl Block { + const fn genesis(leader: ed25519::PublicKey) -> Self { + Self { + context: Context { + round: Round::new(Epoch::zero(), View::zero()), + leader, + parent: (View::zero(), Sha256Digest::EMPTY), + }, + parent: Sha256Digest::EMPTY, + height: Height::zero(), + payload: None, + } + } + + /// Returns the DKG result carried by this block, if present. + pub const fn epoch_info(&self) -> Option<&EpochInfo> { + match &self.payload { + Some(Payload::EpochInfo(info)) => Some(info), + _ => None, + } + } +} + +impl Write for Block { + fn write(&self, buf: &mut impl BufMut) { + self.context.write(buf); + self.parent.write(buf); + self.height.write(buf); + self.payload.write(buf); + } +} + +impl EncodeSize for Block { + fn encode_size(&self) -> usize { + self.context.encode_size() + + self.parent.encode_size() + + self.height.encode_size() + + self.payload.encode_size() + } +} + +impl Read for Block { + type Cfg = NonZeroU32; + + fn read_cfg(buf: &mut impl Buf, max_participants: &Self::Cfg) -> Result { + Ok(Self { + context: Context::read(buf)?, + parent: sha256::Digest::read(buf)?, + height: Height::read(buf)?, + payload: Option::>::read_cfg(buf, max_participants)?, + }) + } +} + +impl Digestible for Block { + type Digest = sha256::Digest; + + fn digest(&self) -> sha256::Digest { + Sha256::hash(&self.encode()) + } +} + +impl Heightable for Block { + fn height(&self) -> Height { + self.height + } +} + +impl ConsensusBlock for Block { + fn parent(&self) -> sha256::Digest { + self.parent + } +} + +impl CertifiableBlock for Block { + type Context = Context; + + fn context(&self) -> Self::Context { + self.context.clone() + } +} + +impl ReshareBlock for Block { + type Variant = V; + type Signer = ed25519::PrivateKey; + + fn payload(&self) -> Option> { + self.payload.clone() + } +} + +/// Self-contained DKG engine. +pub struct Engine +where + V: Variant, +{ + context: ContextCell, + config: Config, + _variant: PhantomData, +} + +impl Engine +where + V: Variant, +{ + /// Creates a new engine. + pub const fn new(context: E, config: Config) -> Self { + Self { + context: ContextCell::new(context), + config, + _variant: PhantomData, + } + } +} + +impl Engine +where + E: CryptoRng + Spawner + Metrics + Clock + Storage + BufferPooler, + V: Variant, + M: Manager + Clone, + X: Blocker + Clone, + SS: SecretStore, + T: Strategy + Clone, + ed25519::Batch: BatchVerifier + Send + 'static, +{ + /// Starts consensus, marshal, broadcast, and the private reshare DKG actor. + #[allow(clippy::type_complexity, clippy::too_many_arguments)] + pub fn start( + mut self, + votes: ( + impl Sender, + impl Receiver, + ), + certificates: ( + impl Sender, + impl Receiver, + ), + resolver: ( + impl Sender, + impl Receiver, + ), + backfill: ( + impl Sender, + impl Receiver, + ), + broadcast: ( + impl Sender, + impl Receiver, + ), + dkg: ( + impl Sender, + impl Receiver, + ), + ) -> (Handle<()>, oneshot::Receiver>) { + let (completion_tx, completion_rx) = oneshot::channel(); + let handle = spawn_cell!( + self.context, + self.run( + votes, + certificates, + resolver, + backfill, + broadcast, + dkg, + completion_tx + ) + ); + (handle, completion_rx) + } + + #[allow(clippy::too_many_arguments)] + async fn run( + self, + votes: ( + impl Sender, + impl Receiver, + ), + certificates: ( + impl Sender, + impl Receiver, + ), + resolver_network: ( + impl Sender, + impl Receiver, + ), + backfill: ( + impl Sender, + impl Receiver, + ), + broadcast: ( + impl Sender, + impl Receiver, + ), + dkg: ( + impl Sender, + impl Receiver, + ), + completion: oneshot::Sender>, + ) { + assert!( + !self.config.participants.is_empty(), + "DKG requires at least one participant" + ); + Participants { + dealers: self.config.participants.clone(), + players: self.config.participants.clone(), + next_players: Set::default(), + } + .validate_epoch_capacity::(self.config.blocks_per_epoch, None) + .expect("DKG epoch must have enough dealer-log slots"); + let participants = self + .config + .participants + .len() + .try_into() + .expect("too many DKG participants"); + let max_participants = NZU32!(participants); + + let context = self.context.into_present(); + let page_cache = CacheRef::from_pooler(&context, PAGE_SIZE, PAGE_CACHE_PAGES); + let public_key = self.config.signer.public_key(); + let consensus_namespace = [self.config.namespace, b"_INITIAL_CONSENSUS"].concat(); + let scheme = ConsensusScheme::signer( + &consensus_namespace, + self.config.participants.clone(), + self.config.signer.clone(), + ) + .expect("DKG signer must be a participant"); + let provider = ConstantProvider::<_, Epoch>::new(scheme.clone()); + let genesis = Block::::genesis( + self.config + .participants + .iter() + .next() + .expect("participants must be non-empty") + .clone(), + ); + + let (buffer, buffer_mailbox) = buffered::Engine::new( + context.child("buffer"), + buffered::Config { + public_key: public_key.clone(), + mailbox_size: MAILBOX_SIZE, + deque_size: 16, + priority: false, + codec_config: max_participants, + peer_provider: self.config.manager.clone(), + }, + ); + let buffer_handle = buffer.start(broadcast); + + let (backfill_handler, backfill_resolver) = marshal_resolver::init( + context.child("backfill"), + marshal_resolver::Config { + public_key: public_key.clone(), + peer_provider: self.config.manager.clone(), + blocker: self.config.blocker.clone(), + mailbox_size: MAILBOX_SIZE, + initial: Duration::from_secs(1), + timeout: Duration::from_secs(2), + fetch_retry_timeout: Duration::from_millis(100), + priority_requests: false, + priority_responses: false, + }, + backfill, + ); + + let finalizations = prunable::Archive::init( + context.child("finalizations"), + archive_config( + &self.config.partition_prefix, + "finalizations", + page_cache.clone(), + ConsensusScheme::certificate_codec_config_unbounded(), + ), + ) + .await + .expect("failed to initialize DKG finalization archive"); + let blocks = prunable::Archive::init( + context.child("blocks"), + archive_config( + &self.config.partition_prefix, + "blocks", + page_cache.clone(), + max_participants, + ), + ) + .await + .expect("failed to initialize DKG block archive"); + + let (marshal_actor, marshal_mailbox, _) = MarshalActor::init( + context.child("marshal"), + finalizations, + blocks, + marshal::Config { + provider: provider.clone(), + epocher: FixedEpocher::new(self.config.blocks_per_epoch), + start: Start::Genesis(genesis.clone()), + partition_prefix: format!("{}-marshal", self.config.partition_prefix), + mailbox_size: MAILBOX_SIZE, + view_retention_timeout: ViewDelta::new(10), + prunable_items_per_section: ARCHIVE_ITEMS_PER_SECTION, + page_cache: page_cache.clone(), + replay_buffer: IO_BUFFER_SIZE, + key_write_buffer: IO_BUFFER_SIZE, + value_write_buffer: IO_BUFFER_SIZE, + block_codec_config: max_participants, + max_repair: NZUsize!(10), + max_pending_acks: NZUsize!(1), + strategy: self.config.strategy.clone(), + }, + ) + .await; + + let (fence, _gate) = Fence::new(Epoch::zero()); + let (reshare_actor, reshare_mailbox) = reshare::Actor::new_dkg( + context.child("reshare"), + reshare::Config { + signer: self.config.signer.clone(), + manager: self.config.manager.clone(), + blocker: self.config.blocker.clone(), + participants_provider: StaticParticipants { + participants: self.config.participants.clone(), + }, + secret_store: self.config.secret_store, + strategy: self.config.strategy.clone(), + registrar: NoopRegistrar(PhantomData), + marshal: marshal_mailbox.clone(), + state_sync_floor: None, + fence, + namespace: self.config.namespace, + sharing_mode: self.config.sharing_mode, + mailbox_size: MAILBOX_SIZE, + partition_prefix: format!("{}-reshare", self.config.partition_prefix), + max_participants, + blocks_per_epoch: self.config.blocks_per_epoch, + batch_verifier: PhantomData::, + }, + DkgConfig { + participants: self.config.participants.clone(), + completion: Box::new(move |info| { + let _ = completion.send_lossy(Completion { info }); + }), + }, + ); + + let app = reshare::Application::new( + DkgApp(PhantomData), + reshare_mailbox.clone(), + self.config.blocks_per_epoch, + ); + let deferred = Deferred::new( + context.child("deferred"), + app, + marshal_mailbox.clone(), + FixedEpocher::new(self.config.blocks_per_epoch), + ); + let simplex = simplex::Engine::new( + context.child("simplex"), + simplex::Config { + scheme, + elector: RoundRobin::::default(), + blocker: self.config.blocker, + automaton: deferred.clone(), + relay: deferred, + reporter: marshal_mailbox.clone(), + strategy: self.config.strategy, + partition: format!("{}-simplex", self.config.partition_prefix), + mailbox_size: MAILBOX_SIZE, + epoch: Epoch::zero(), + floor: Floor::Genesis(genesis.digest()), + replay_buffer: IO_BUFFER_SIZE, + write_buffer: IO_BUFFER_SIZE, + page_cache, + leader_timeout: Duration::from_secs(1), + certification_timeout: Duration::from_secs(2), + timeout_retry: Duration::from_millis(500), + activity_timeout: ViewDelta::new(10), + skip_timeout: ViewDelta::new(5), + fetch_timeout: Duration::from_secs(2), + fetch_concurrent: NZUsize!(4), + forwarding: ForwardingPolicy::Disabled, + }, + ); + + let reshare_handle = reshare_actor.start(dkg); + let marshal_handle = marshal_actor.start( + reshare_mailbox, + buffer_mailbox, + (backfill_handler, backfill_resolver), + ); + let simplex_handle = simplex.start(votes, certificates, resolver_network); + + BootstrapActors { + buffer_handle, + reshare_handle, + marshal_handle, + simplex_handle, + } + .supervise() + .await; + } +} + +struct BootstrapActors { + buffer_handle: Handle<()>, + reshare_handle: Handle<()>, + marshal_handle: Handle<()>, + simplex_handle: Handle<()>, +} + +impl BootstrapActors { + async fn supervise(mut self) { + select! { + result = &mut self.buffer_handle => result.expect("failed dkg"), + result = &mut self.reshare_handle => result.expect("failed dkg"), + result = &mut self.marshal_handle => result.expect("failed dkg"), + result = &mut self.simplex_handle => result.expect("failed dkg"), + } + } +} + +impl Drop for BootstrapActors { + fn drop(&mut self) { + self.buffer_handle.abort(); + self.reshare_handle.abort(); + self.marshal_handle.abort(); + self.simplex_handle.abort(); + } +} + +#[derive(Clone)] +struct DkgApp(PhantomData); + +impl Application for DkgApp +where + E: Rng + Spawner + Metrics + Clock, + V: Variant, +{ + type SigningScheme = ConsensusScheme; + type Context = Context; + type Block = Block; + type Input = reshare::Input<(), V, ed25519::PrivateKey>; + + async fn propose( + &mut self, + (_, context): (E, Self::Context), + ancestry: impl Ancestry, + input: Self::Input, + ) -> Option { + let parent = ancestry.peek()?.clone(); + let height = parent.height().next(); + Some(Block { + context, + parent: parent.digest(), + height, + payload: input.payload, + }) + } + + async fn verify( + &mut self, + _: (E, Self::Context), + _ancestry: impl Ancestry, + ) -> bool { + true + } +} + +#[derive(Clone)] +struct StaticParticipants

{ + participants: Set

, +} + +impl

ParticipantsProvider for StaticParticipants

+where + P: PublicKey, +{ + type PublicKey = P; + + async fn participants(&mut self, _: Epoch) -> Set { + self.participants.clone() + } +} + +#[derive(Clone)] +struct NoopRegistrar(PhantomData<(V, P)>); + +impl Registrar for NoopRegistrar +where + V: Variant, + P: PublicKey, +{ + type Variant = V; + type PublicKey = P; + + async fn register(&self, _: Epoch, _: SchemeInfo) {} +} + +fn archive_config( + prefix: &str, + name: &str, + page_cache: CacheRef, + codec_config: C, +) -> prunable::Config { + prunable::Config { + translator: TwoCap, + key_partition: format!("{prefix}-{name}-key"), + key_page_cache: page_cache, + value_partition: format!("{prefix}-{name}-value"), + compression: None, + codec_config, + items_per_section: ARCHIVE_ITEMS_PER_SECTION, + key_write_buffer: IO_BUFFER_SIZE, + value_write_buffer: IO_BUFFER_SIZE, + replay_buffer: IO_BUFFER_SIZE, + } +} diff --git a/glue/src/dkg/fence.rs b/glue/src/dkg/fence.rs new file mode 100644 index 00000000000..43259f0787f --- /dev/null +++ b/glue/src/dkg/fence.rs @@ -0,0 +1,245 @@ +//! Epoch readiness gate used to synchronize the [`Provider`] and the [`orchestrator::Actor`]. +//! +//! [`Provider`]: commonware_cryptography::certificate::Provider +//! [`orchestrator::Actor`]: super::orchestrator::Actor + +use commonware_consensus::types::Epoch; +use futures::task::AtomicWaker; +use std::{ + future::Future, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + task::{Context, Poll}, +}; + +/// Epoch producer dropped before the requested epoch became available. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("epoch fence closed")] +pub struct Closed; + +pub struct Fence { + state: Arc, +} + +impl Fence { + pub fn new(epoch: Epoch) -> (Self, Gate) { + let state = Arc::new(State::new(epoch)); + ( + Self { + state: state.clone(), + }, + Gate { state }, + ) + } + + pub fn epoch(&self) -> Epoch { + self.state.epoch() + } + + pub fn mark(&self, epoch: Epoch) -> Epoch { + self.state.mark(epoch) + } +} + +impl Drop for Fence { + fn drop(&mut self) { + self.state.close(); + } +} + +pub struct Gate { + state: Arc, +} + +impl Gate { + pub fn epoch(&self) -> Epoch { + self.state.epoch() + } + + /// Wait for `epoch` to become available. + /// + /// Returns [`Closed`] if the producer is dropped before the gate reaches the + /// requested epoch. Already-reached epochs still resolve successfully after + /// closure. + pub const fn wait(&mut self, epoch: Epoch) -> Waiter<'_> { + Waiter { gate: self, epoch } + } +} + +pub struct Waiter<'a> { + gate: &'a Gate, + epoch: Epoch, +} + +impl Future for Waiter<'_> { + type Output = Result<(), Closed>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.gate.state.waker.register(cx.waker()); + + let closed = self.gate.state.closed.load(Ordering::Acquire); + if self.epoch <= self.gate.state.epoch() { + return Poll::Ready(Ok(())); + } + + if closed { + Poll::Ready(Err(Closed)) + } else { + Poll::Pending + } + } +} + +struct State { + epoch: AtomicU64, + closed: AtomicBool, + waker: AtomicWaker, +} + +impl State { + const fn new(epoch: Epoch) -> Self { + Self { + epoch: AtomicU64::new(epoch.get()), + closed: AtomicBool::new(false), + waker: AtomicWaker::new(), + } + } + + fn epoch(&self) -> Epoch { + Epoch::new(self.epoch.load(Ordering::Acquire)) + } + + fn mark(&self, epoch: Epoch) -> Epoch { + let previous = self.epoch.fetch_max(epoch.get(), Ordering::AcqRel); + let latest = Epoch::new(previous.max(epoch.get())); + if epoch.get() > previous { + self.waker.wake(); + } + latest + } + + fn close(&self) { + self.closed.store(true, Ordering::Release); + self.waker.wake(); + } +} + +#[cfg(test)] +mod tests { + use super::{Closed, Fence}; + use commonware_consensus::types::Epoch; + use commonware_macros::test_async; + use futures::task::{ArcWake, waker_ref}; + use std::{ + future::Future, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + task::{Context, Poll}, + }; + + struct WakeCounter(AtomicUsize); + + impl WakeCounter { + fn new() -> Arc { + Arc::new(Self(AtomicUsize::new(0))) + } + + fn count(&self) -> usize { + self.0.load(Ordering::Relaxed) + } + } + + impl ArcWake for WakeCounter { + fn wake_by_ref(arc_self: &Arc) { + arc_self.0.fetch_add(1, Ordering::Relaxed); + } + } + + #[test_async] + async fn resolves_immediately_for_ready_epoch() { + let (_fence, mut gate) = Fence::new(Epoch::new(2)); + gate.wait(Epoch::new(2)).await.unwrap(); + } + + #[test_async] + async fn resolves_after_mark() { + let (fence, mut gate) = Fence::new(Epoch::zero()); + assert_eq!(fence.mark(Epoch::new(1)), Epoch::new(1)); + assert_eq!(fence.epoch(), Epoch::new(1)); + assert_eq!(gate.epoch(), Epoch::new(1)); + + gate.wait(Epoch::new(1)).await.unwrap(); + } + + #[test_async] + async fn resolves_sequential_waiters() { + let (fence, mut gate) = Fence::new(Epoch::zero()); + + let first = gate.wait(Epoch::new(1)); + fence.mark(Epoch::new(1)); + first.await.unwrap(); + + let second = gate.wait(Epoch::new(2)); + fence.mark(Epoch::new(2)); + second.await.unwrap(); + } + + #[test] + fn waits_for_requested_epoch() { + let (fence, mut gate) = Fence::new(Epoch::zero()); + let mut waiter = Box::pin(gate.wait(Epoch::new(2))); + let second_wakes = WakeCounter::new(); + + let second_waker = waker_ref(&second_wakes); + let mut second_context = Context::from_waker(&second_waker); + assert!(waiter.as_mut().poll(&mut second_context).is_pending()); + + fence.mark(Epoch::new(1)); + + assert!(waiter.as_mut().poll(&mut second_context).is_pending()); + + fence.mark(Epoch::new(2)); + + assert!(waiter.as_mut().poll(&mut second_context).is_ready()); + assert!(second_wakes.count() > 0); + } + + #[test] + fn producer_drop_wakes_waiter_with_closed() { + let (fence, mut gate) = Fence::new(Epoch::zero()); + let mut waiter = Box::pin(gate.wait(Epoch::new(1))); + let wakes = WakeCounter::new(); + + let waker = waker_ref(&wakes); + let mut context = Context::from_waker(&waker); + assert!(waiter.as_mut().poll(&mut context).is_pending()); + + drop(fence); + + assert_eq!(wakes.count(), 1); + assert_eq!(waiter.as_mut().poll(&mut context), Poll::Ready(Err(Closed))); + } + + #[test_async] + async fn ready_epoch_still_resolves_after_producer_drop() { + let (fence, mut gate) = Fence::new(Epoch::new(1)); + + drop(fence); + + gate.wait(Epoch::new(1)).await.unwrap(); + } + + #[test_async] + async fn mark_does_not_regress_epoch() { + let (fence, mut gate) = Fence::new(Epoch::new(2)); + + assert_eq!(fence.mark(Epoch::new(1)), Epoch::new(2)); + assert_eq!(fence.epoch(), Epoch::new(2)); + gate.wait(Epoch::new(2)).await.unwrap(); + } +} diff --git a/glue/src/dkg/mod.rs b/glue/src/dkg/mod.rs new file mode 100644 index 00000000000..d23bbf8b83b --- /dev/null +++ b/glue/src/dkg/mod.rs @@ -0,0 +1,245 @@ +//! Bootstrap and continuously reshare threshold secrets. +//! +//! This module wires threshold-key management into consensus without owning the +//! application's state machine or private-key policy. It provides two public +//! entry points: +//! +//! - [`bootstrap`] runs a contained, one-shot DKG chain that trustlessly creates +//! an initial threshold secret. +//! - [`reshare`] runs alongside an application chain and continuously rotates +//! threshold shares across epochs. +//! +//! Both paths produce or consume [`types::EpochInfo`], the public artifact that +//! describes the threshold output for an epoch. The application stores that +//! artifact in its own blocks and installs epoch-scoped schemes through a +//! [`Registrar`]. +//! +//! # Application Contract +//! +//! Application blocks implement [`ReshareBlock`] and carry at most one +//! [`types::Payload`]. Connect an application to the reshare mailbox by wrapping +//! it in [`reshare::Application`], which drives both sides of the contract: +//! +//! - For proposals, the wrapper selects and fetches the payload to include (a +//! dealer log from the midpoint onward, the epoch info on the final block) and +//! hands it to the application through [`reshare::Input`]. The +//! application takes it in its own `propose` and attaches it to the block it +//! builds, because only the application can build its block type. It does not +//! talk to the reshare mailbox or track epoch boundaries itself. +//! - For verification, the wrapper rejects a final block whose payload does not +//! match the independently constructed [`types::EpochInfo`], and rejects stray +//! payloads on early non-final blocks, so the application does not implement +//! these checks by hand. +//! +//! The protocol also requires the application to provide a [`SecretStore`]. +//! Secret storage is intentionally user-owned: deployments differ on encryption, +//! access control, hardware isolation, backups, and pruning. Anything written to +//! this trait is private ceremony material and must be protected by the +//! application's security policy. +//! +//! # State Sync +//! +//! Reshare supports nodes that join through state sync. A syncing node can only +//! participate in a future ceremony if it syncs while it is a `next_player`. +//! Once the node becomes a `player`, it must already be online for the early +//! ceremony traffic. Syncing during that epoch is too late to receive a private +//! share, and the protocol will reveal that share instead. +//! +//! This timing makes it safe for [`ParticipantsProvider`] to be backed by chain +//! state (e.g., a staking contract). The chain can announce future players first, +//! giving those nodes an epoch to state sync and enter the next ceremony normally. +//! +//! # Marshal Retention +//! +//! DKG startup relies on marshal's local finalized block archive unless the node +//! is entering through one-time state sync. On an ordinary restart, the active +//! epoch is derived from marshal's processed height, and the public +//! [`types::EpochInfo`] for that epoch is loaded from the finalized boundary +//! block that introduced it. +//! +//! For epoch zero, that boundary is height zero. For later epochs, the boundary +//! is the final block of the previous epoch: +//! +//! ```text +//! boundary(current_epoch) = last_block(current_epoch - 1) +//! ``` +//! +//! An operator running stateful pruning MUST keep marshal's finalized block +//! retention window at least one full epoch wide, so the previous epoch's +//! boundary block survives until the current epoch finishes. Concretely, the +//! marshal retention floor configured through the stateful +//! [`PruneConfig`](crate::stateful::PruneConfig) +//! (`max_pending_acks + 1 + retained_marshal_blocks` finalized blocks) MUST be +//! greater than or equal to the DKG epoch length (`blocks_per_epoch`). DKG does +//! not need blocks before that previous boundary for ordinary restart, but it +//! does need the boundary block itself to recover the epoch's public threshold +//! output, participant set, and Simplex floor commitment. +//! +//! This coupling is the operator's responsibility. The two knobs are configured +//! independently: `blocks_per_epoch` is a DKG configuration, while the marshal +//! retention floor is set on the stateful +//! [`PruneConfig`](crate::stateful::PruneConfig). The library cannot enforce the +//! relationship, and no runtime check couples them +//! ([`PruneConfig::assert_valid`](crate::stateful::PruneConfig::assert_valid) +//! only compares marshal and QMDB retention). Pruning the boundary before the +//! current epoch finishes leaves a restarting validator without the local public +//! material required for normal recovery, and the orchestrator panics on startup +//! with a `missing finalized boundary block` error. +//! +//! Nodes that serve `dkg::anchor` responses for other peers also need the +//! corresponding boundary finalization and boundary block for every epoch they +//! intend to serve. +//! +//! See [`anchor`], [`fence`], [`orchestrator`], [`reshare`], and [`types`] for +//! the detailed actors, synchronization points, and wire artifacts. + +use crate::dkg::types::SchemeInfo; +use commonware_consensus::{Block, types::Epoch}; +use commonware_cryptography::{ + PublicKey, Signer, + bls12381::{ + dkg::feldman_desmedt::DealerPrivMsg, + primitives::{group::Share, variant::Variant}, + }, + transcript::Summary, +}; +use commonware_utils::ordered::Set; +use std::future::Future; + +pub mod anchor; +pub mod bootstrap; +pub mod fence; +pub mod orchestrator; +pub mod reshare; +pub mod types; + +#[cfg(test)] +mod tests; + +/// A [`Block`] that may carry a reshare [`Payload`](types::Payload). +pub trait ReshareBlock: Block { + /// BLS variant used by the DKG payload. + type Variant: Variant; + + /// Signer type used by DKG payloads. + type Signer: Signer; + + /// Retrieves the [`Payload`](types::Payload) carried by this block, if any. + fn payload(&self) -> Option>; +} + +/// A registrar of signing schemes that supplies a [`Provider`] an [`Epoch`]-scoped +/// [`ThresholdScheme`] in preparation for a transition to the given [`Epoch`]. +/// +/// [`Provider`]: commonware_cryptography::certificate::Provider +/// [`ThresholdScheme`]: commonware_consensus::simplex::scheme::bls12381_threshold +pub trait Registrar: Send + Sync + 'static { + /// BLS variant used by the DKG payload. + type Variant: Variant; + + /// Participant public key type. + type PublicKey: PublicKey; + + /// Hook for handling an epoch transition. + fn register( + &self, + epoch: Epoch, + info: SchemeInfo, + ) -> impl Future + Send; +} + +/// Interface for a secret store that persists and retrieves the private DKG/reshare +/// material for different [`Epoch`]s. +/// +/// All material entrusted to this trait is secret and must be stored as such: it must +/// never be written to plaintext protocol storage, carried on-chain, or sent to peers. +/// This includes the dealer RNG seed, which seeds a dealer's sharing polynomial and so +/// reveals every share that dealer sends. +/// +/// Writes must be durable before their returned future resolves. When +/// [`put_share`](Self::put_share), [`put_seed`](Self::put_seed), or +/// [`put_dealing`](Self::put_dealing) resolves, the stored material MUST survive a crash: the +/// reshare actor treats a resolved put as a durable commitment and does not re-derive the +/// material after a restart. A buffered store that resolves before the write is stable can let a +/// dealer reseed with fresh randomness and re-deal different shares for the same epoch +/// (equivocation), or lose a share it has already relied upon. +pub trait SecretStore: Send + Sync + 'static { + /// Stores a [`Share`] for a given [`Epoch`]. + /// + /// Must be durable before the returned future resolves (see the trait documentation). + fn put_share(&mut self, epoch: Epoch, share: Share) -> impl Future + Send; + + /// Retrieves a [`Share`] for a given [`Epoch`], if it exists. + fn get_share(&mut self, epoch: Epoch) -> impl Future> + Send; + + /// Stores the dealer RNG seed for a given [`Epoch`]. + /// + /// The seed deterministically replays this node's dealer randomness across a + /// restart. It is secret: knowing it reveals every share the dealer distributes. + /// + /// Must be durable before the returned future resolves: no-equivocation safety depends on the + /// seed being recovered verbatim after a crash so the dealer replays identical randomness + /// rather than re-dealing fresh shares. + fn put_seed(&mut self, epoch: Epoch, seed: Summary) -> impl Future + Send; + + /// Retrieves the dealer RNG seed for a given [`Epoch`], if it exists. + fn get_seed(&mut self, epoch: Epoch) -> impl Future> + Send; + + /// Stores a private dealing received from `dealer` during `epoch`. + fn put_dealing( + &mut self, + epoch: Epoch, + dealer: P, + private: DealerPrivMsg, + ) -> impl Future + Send; + + /// Retrieves a private dealing received from `dealer` during `epoch`. + fn get_dealing( + &mut self, + epoch: Epoch, + dealer: &P, + ) -> impl Future> + Send; + + /// Prunes secrets older than `min`. + fn prune(&mut self, min: Epoch) -> impl Future + Send; +} + +/// Participant policy provider. +/// +/// This is the only application hook on canonical epoch structure: it supplies +/// the intended participant set for a future `epoch`. The actor derives dealers, +/// current players, and ordinary epoch progression from finalized public truth, +/// and consults this only for the players of an epoch it cannot yet read from a +/// finalized boundary block. +/// +/// [`participants`](Self::participants) must be deterministic at a given epoch +/// across all honest nodes (see its documentation for the exact contract). +pub trait ParticipantsProvider: Send + Sync + 'static { + type PublicKey: PublicKey; + + /// Returns the intended participant set for `epoch`. + /// + /// This MUST be deterministic and stable: for a given `epoch`, every honest + /// node MUST return an identical [`Set`], with the same membership AND the + /// same ordering, and repeated calls MUST return the same `Set`. + /// + /// The returned set MUST be non-empty and MUST contain no more than the + /// actor's configured `max_participants` entries. A violation is treated as + /// a deterministic provider contract failure. + /// + /// In continuous reshare, this hook is consulted while building or + /// verifying an epoch's final block. That final block carries the + /// [`types::EpochInfo`] for the next epoch, and this set is embedded + /// verbatim as that epoch info's `next_players`. + /// + /// Therefore the result for `epoch` must be locked in before honest nodes + /// propose or verify the final block that announces `epoch` as + /// `next_players`. The proposer and every verifier independently rebuild + /// and compare the value for equality. Because [`Set`] is order sensitive + /// (both its equality and its encoding depend on element order), any + /// divergence in membership or ordering between proposer and verifier + /// rejects a valid final block and stalls the epoch boundary. Canonicalize + /// (e.g. sort) the returned `Set` so it is identical regardless of how the + /// underlying membership is stored or queried. + fn participants(&mut self, epoch: Epoch) -> impl Future> + Send; +} diff --git a/glue/src/dkg/orchestrator/actor.rs b/glue/src/dkg/orchestrator/actor.rs new file mode 100644 index 00000000000..1c5ac200874 --- /dev/null +++ b/glue/src/dkg/orchestrator/actor.rs @@ -0,0 +1,769 @@ +//! Consensus engine orchestration for threshold reshare epoch transitions. + +use crate::dkg::{ + ReshareBlock, anchor, + fence::Gate, + orchestrator::{Mailbox, mailbox::Message}, + types::{EpochInfo, Payload}, +}; +use commonware_actor::mailbox; +use commonware_consensus::{ + CertifiableAutomaton, Epochable, Heightable, Relay, + marshal::core::{Mailbox as MarshalMailbox, Variant as MarshalVariant}, + simplex::{ + self, Floor, ForwardingPolicy, Plan, + elector::Config as Elector, + scheme, + types::{Context, Finalization}, + }, + types::{Epoch, Epocher, FixedEpocher, Height, ViewDelta}, +}; +use commonware_cryptography::{ + Digest, PublicKey, Signer, + bls12381::primitives::variant::Variant as BlsVariant, + certificate::{Provider, Verifier}, +}; +use commonware_macros::{select, select_loop}; +use commonware_p2p::{ + Blocker, Channel, Manager, Message as P2pMessage, Receiver, Sender, TrackedPeers, + utils::mux::{Builder, MuxHandle, Muxer}, +}; +use commonware_parallel::Strategy; +use commonware_runtime::{ + BufferPooler, Clock, ContextCell, Handle, Metrics, Network, Spawner, Storage, + buffer::paged::CacheRef, + spawn_cell, + telemetry::metrics::{Gauge, GaugeExt, MetricsExt as _}, +}; +use commonware_utils::{Acknowledgement, acknowledgement::Exact, channel::mpsc, vec::NonEmptyVec}; +use futures::FutureExt as _; +use rand_core::CryptoRng; +use std::{ + marker::PhantomData, + num::{NonZeroU16, NonZeroU64, NonZeroUsize}, + sync::Arc, + time::Duration, +}; +use tracing::{debug, info}; + +/// Offset leaves peer-set ID zero available for anchor bootstrap peers. +const EPOCH_PEER_SET_OFFSET: u64 = 1; + +struct Channels +where + C: Verifier, + S: Sender, + R: Receiver, +{ + vote: MuxHandle, + vote_backup: mpsc::Receiver<(Channel, P2pMessage)>, + certificate: MuxHandle, + resolver: MuxHandle, +} + +struct ActiveEpoch { + epoch: Epoch, + handle: Handle<()>, +} + +impl Drop for ActiveEpoch { + fn drop(&mut self) { + self.handle.abort(); + } +} + +enum EnterEpochError { + GateClosed, + MuxClosed, + Stopped, +} + +/// Public boundary material used for one-time state-sync startup. +pub struct StateSync +where + S: scheme::Scheme, + D: Digest, + V: BlsVariant, +{ + /// Public boundary material discovered by `dkg::anchor`. + pub artifact: anchor::Artifact, + + /// Finalized floor selected by `stateful::probe`. + pub floor: Finalization, +} + +struct ResolvedStart +where + S: scheme::Scheme, + D: Digest, + V: BlsVariant, + P: PublicKey, +{ + epoch: Epoch, + floor: Floor, + info: EpochInfo, +} + +/// Simplex configuration applied to each epoch engine. +#[derive(Clone)] +pub struct SimplexConfig { + /// Leader election configuration. + pub elector: L, + + /// Maximum number of messages to buffer on channels inside each consensus engine. + pub mailbox_size: NonZeroUsize, + + /// Number of bytes to buffer when replaying consensus state during startup. + pub replay_buffer: NonZeroUsize, + + /// Number of bytes to buffer when writing consensus journal blobs. + pub write_buffer: NonZeroUsize, + + /// Page size used by the consensus journal page cache. + pub page_cache_page_size: NonZeroU16, + + /// Number of pages retained by the consensus journal page cache. + pub page_cache_pages: NonZeroUsize, + + /// Time to wait for a leader proposal in a view. + pub leader_timeout: Duration, + + /// Time to wait for certification progress before attempting to skip a view. + pub certification_timeout: Duration, + + /// Time to wait before retrying a nullify broadcast while stuck in a view. + pub timeout_retry: Duration, + + /// Time to wait for a peer to respond to a resolver request. + pub fetch_timeout: Duration, + + /// Number of concurrent resolver requests. + pub fetch_concurrent: NonZeroUsize, + + /// Number of views behind the finalized tip to retain validator activity. + pub activity_timeout: ViewDelta, + + /// Recent inactive leader window that triggers immediate nullification. + pub skip_timeout: ViewDelta, + + /// Policy for proactively forwarding certified blocks. + pub forwarding: ForwardingPolicy, +} + +/// Configuration for the [`Actor`]. +pub struct Config +where + P: Provider, + P::Scheme: scheme::Scheme, + MV: MarshalVariant, + DV: BlsVariant, +{ + /// Network blocker shared with each epoch consensus engine. + pub oracle: B, + + /// P2P manager used to track the active consensus peer set. + pub manager: M, + + /// Provider of epoch-scoped consensus signing schemes. + pub provider: P, + + /// Marshal mailbox used to report consensus output and read finalized blocks. + pub marshal: MarshalMailbox, + + /// Application automaton and relay used by each epoch consensus engine. + pub application: A, + + /// Strategy for parallel verification and signing work. + pub strategy: T, + + /// Simplex settings applied to every epoch engine. + pub simplex: SimplexConfig, + + /// Gate for waiting for the signature scheme to be configured prior to + /// entering an epoch. + pub gate: Gate, + + /// Public boundary material used only when entering through state sync. + pub state_sync: Option>, + + /// Number of blocks in each epoch. + pub blocks_per_epoch: NonZeroU64, + + /// Maximum number of messages to buffer in each network muxer. + pub muxer_size: usize, + + /// Maximum number of finalized-block reports to buffer. + pub mailbox_size: NonZeroUsize, + + /// Partition prefix used for per-epoch consensus persistence. + pub partition_prefix: String, +} + +/// Consensus engine orchestrator. +pub struct Actor +where + E: BufferPooler + Spawner + Metrics + CryptoRng + Clock + Storage + Network, + B: Blocker::PublicKey>, + M: Manager::PublicKey>, + P: Provider, + P::Scheme: scheme::Scheme, + MV: MarshalVariant, + MV::ApplicationBlock: ReshareBlock, + DV: BlsVariant, + C: Signer::PublicKey>, + A: CertifiableAutomaton< + Context = Context::PublicKey>, + Digest = MV::Commitment, + > + Relay< + Digest = MV::Commitment, + PublicKey = ::PublicKey, + Plan = Plan<::PublicKey>, + >, + L: Elector, + T: Strategy, + ACK: Acknowledgement, +{ + context: ContextCell, + mailbox: mailbox::Receiver>, + oracle: B, + manager: M, + provider: P, + marshal: MarshalMailbox, + application: A, + strategy: T, + simplex: SimplexConfig, + gate: Gate, + state_sync: Option>, + blocks_per_epoch: NonZeroU64, + muxer_size: usize, + partition_prefix: String, + page_cache_ref: CacheRef, + latest_epoch: Gauge, + _payload: PhantomData<(DV, C)>, +} + +impl Actor +where + E: BufferPooler + Spawner + Metrics + CryptoRng + Clock + Storage + Network, + B: Blocker::PublicKey>, + M: Manager::PublicKey>, + P: Provider, + P::Scheme: scheme::Scheme, + MV: MarshalVariant, + MV::ApplicationBlock: ReshareBlock, + DV: BlsVariant, + C: Signer::PublicKey>, + A: CertifiableAutomaton< + Context = Context::PublicKey>, + Digest = MV::Commitment, + > + Relay< + Digest = MV::Commitment, + PublicKey = ::PublicKey, + Plan = Plan<::PublicKey>, + >, + L: Elector, + T: Strategy, + ACK: Acknowledgement, +{ + /// Build an orchestrator and the mailbox that receives finalized blocks. + /// + /// The returned [`Mailbox`] should be installed as a marshal reporter. The + /// actor uses those finalized-block reports to advance epochs after it is + /// spawned with [`Actor::start`]. + pub fn new( + context: E, + config: Config, + ) -> (Self, Mailbox) { + if let Some(state_sync) = &config.state_sync { + assert_eq!( + state_sync.artifact.epoch, + state_sync.floor.epoch(), + "state sync artifact and floor must be in the same epoch" + ); + } + + let (sender, mailbox) = mailbox::new(context.child("mailbox"), config.mailbox_size); + let page_cache_ref = CacheRef::from_pooler( + &context, + config.simplex.page_cache_page_size, + config.simplex.page_cache_pages, + ); + let latest_epoch = context.gauge("latest_epoch", "current epoch"); + + ( + Self { + context: ContextCell::new(context), + mailbox, + oracle: config.oracle, + manager: config.manager, + provider: config.provider, + marshal: config.marshal, + application: config.application, + strategy: config.strategy, + simplex: config.simplex, + gate: config.gate, + state_sync: config.state_sync, + blocks_per_epoch: config.blocks_per_epoch, + muxer_size: config.muxer_size, + partition_prefix: config.partition_prefix, + page_cache_ref, + latest_epoch, + _payload: PhantomData, + }, + Mailbox::new(sender), + ) + } + + /// Spawn the orchestrator with the consensus network channels. + /// + /// Vote and resolver channels are multiplexed by epoch inside the actor. + /// The certificate mux must already be running so other actors can observe + /// unregistered certificate subchannels before the orchestrator enters an + /// epoch. + pub fn start( + mut self, + votes: (S, R), + certificates: MuxHandle, + resolver: (S, R), + ) -> Handle<()> + where + S: Sender::PublicKey>, + R: Receiver::PublicKey>, + { + spawn_cell!(self.context, self.run(votes, certificates, resolver,)) + } + + /// Run the actor event loop. + /// + /// The loop owns one active Simplex engine at a time. It listens for + /// finalized boundary blocks from marshal and for backup vote traffic from + /// future epochs, which is used only to ask marshal for the missing boundary + /// finalization. + async fn run( + mut self, + (vote_sender, vote_receiver): (S, R), + certificates: MuxHandle, + (resolver_sender, resolver_receiver): (S, R), + ) where + S: Sender::PublicKey>, + R: Receiver::PublicKey>, + { + let mut channels = self.create_channels( + (vote_sender, vote_receiver), + certificates, + (resolver_sender, resolver_receiver), + ); + let epocher = FixedEpocher::new(self.blocks_per_epoch); + let Some(start) = self.resolve_start(&epocher).await else { + debug!("context shutdown while resolving startup epoch"); + return; + }; + let mut active = match self + .enter_epoch( + start.epoch, + start.floor, + start.info.participants().tracked_peers(), + &mut channels, + ) + .await + { + Ok(active) => active, + Err(EnterEpochError::GateClosed) => { + debug!( + epoch = start.epoch.get(), + "epoch gate closed before startup" + ); + return; + } + Err(EnterEpochError::MuxClosed) => { + debug!( + epoch = start.epoch.get(), + "consensus mux closed before startup epoch" + ); + return; + } + Err(EnterEpochError::Stopped) => { + debug!("context shutdown before startup epoch"); + return; + } + }; + + select_loop! { + self.context, + on_stopped => { + debug!("context shutdown, stopping orchestrator"); + }, + Some((their_epoch, (from, _))) = channels.vote_backup.recv() else { + debug!("vote mux backup channel closed, shutting down orchestrator"); + break; + } => { + self.handle_backup_vote(&epocher, active.epoch, their_epoch, from); + }, + result = &mut active.handle => match result { + Ok(()) => { + debug!(epoch = active.epoch.get(), "simplex engine stopped, shutting down orchestrator"); + break; + } + Err(error) => { + panic!("simplex engine for epoch {} stopped unexpectedly: {error}", active.epoch); + } + }, + Some(message) = self.mailbox.recv() else { + debug!("mailbox closed, shutting down orchestrator"); + break; + } => match message { + Message::Finalized { + block, + acknowledgement, + } => { + let keep_running = self + .handle_finalized( + &epocher, + &mut active, + block, + acknowledgement, + &mut channels, + ) + .await; + if !keep_running { + break; + } + } + }, + } + } + + /// Resolve the first epoch this process should run. + /// + /// Normal startup resolves from marshal's local boundary blocks. State-sync + /// startup is the only exception: the node may know a recent public + /// boundary from `dkg::anchor` before it has the previous boundary block in + /// local marshal storage. + /// + /// Returns `None` when marshal becomes unavailable because the context is + /// shutting down. + async fn resolve_start( + &mut self, + epocher: &FixedEpocher, + ) -> Option::PublicKey>> + { + if let Some(state_sync) = &self.state_sync { + return Some(ResolvedStart { + epoch: state_sync.artifact.epoch, + floor: Floor::Finalized(state_sync.floor.clone()), + info: state_sync.artifact.info.clone(), + }); + } + + let epoch = + self.marshal + .get_processed_height() + .await + .map_or_else(Epoch::zero, |processed| { + let height = processed.next(); + epocher + .containing(height) + .expect("epocher must know recovered height") + .epoch() + }); + self.resolve_boundary(epoch, epocher).await + } + + /// Resolve a locally recovered epoch from marshal's finalized boundary block. + /// + /// Ordinary restarts should not re-enter the configured bootstrap epoch if + /// marshal has already delivered finalized blocks to the application. The + /// processed height names the next block marshal will deliver; from that + /// height we derive the active epoch, then read the boundary block that + /// carried that epoch's public [`EpochInfo`]. That boundary block supplies + /// both the Simplex floor commitment and the peer set to track for the + /// recovered epoch. + /// + /// This is intentionally not used for state-sync startup: during one-time + /// state sync, marshal is anchored at a probe-selected finalization while the + /// previous epoch boundary block is not locally available yet. In that + /// startup path, the anchor artifact is the trusted source of boundary + /// epoch info. + /// + /// Returns `None` when marshal becomes unavailable because the context is + /// shutting down. + async fn resolve_boundary( + &mut self, + epoch: Epoch, + epocher: &FixedEpocher, + ) -> Option::PublicKey>> + { + let height = epoch + .previous() + .and_then(|epoch| epocher.last(epoch)) + .unwrap_or_else(Height::zero); + let Some(boundary) = self.marshal.get_block(height).await else { + // Marshal cancels pending reads when it stops; only an + // intact-but-absent boundary block is a retention violation. + if self.context.stopped().now_or_never().is_some() { + debug!(%height, "boundary block read canceled during shutdown"); + return None; + } + panic!("missing finalized boundary block at height {height}"); + }; + let commitment = MV::commitment(&boundary); + let block = MV::into_inner(boundary); + let Some(Payload::EpochInfo(info)) = block.payload() else { + panic!("boundary block {height} missing epoch info"); + }; + if info.epoch != epoch { + panic!( + "boundary block {height} carries epoch info for {}, expected {epoch}", + info.epoch + ); + } + + Some(ResolvedStart { + epoch, + floor: Floor::Genesis(commitment), + info, + }) + } + + /// Start the consensus channel muxers and return handles used to open + /// epoch-specific subchannels. + /// + /// The vote mux includes a backup receiver so the orchestrator can detect + /// messages for epochs it has not registered locally. + fn create_channels( + &self, + (vote_sender, vote_receiver): (S, R), + certificate: MuxHandle, + (resolver_sender, resolver_receiver): (S, R), + ) -> Channels + where + S: Sender::PublicKey>, + R: Receiver::PublicKey>, + { + let (mux, vote, vote_backup) = Muxer::builder( + self.context.child("vote_mux"), + vote_sender, + vote_receiver, + self.muxer_size, + ) + .with_backup() + .build(); + mux.start(); + + let (mux, resolver) = Muxer::new( + self.context.child("resolver_mux"), + resolver_sender, + resolver_receiver, + self.muxer_size, + ); + mux.start(); + + Channels { + vote, + vote_backup, + certificate, + resolver, + } + } + + /// Handle traffic for an epoch whose vote subchannel is not registered. + /// + /// Messages from past or current epochs are ignored. A future-epoch vote is + /// evidence that peers have crossed an epoch boundary locally, so the actor + /// hints marshal to fetch the current epoch's boundary finalization from the + /// sender. + fn handle_backup_vote( + &self, + epocher: &FixedEpocher, + our_epoch: Epoch, + their_epoch: u64, + from: ::PublicKey, + ) { + let their_epoch = Epoch::new(their_epoch); + if their_epoch <= our_epoch { + debug!(%their_epoch, %our_epoch, ?from, "received message from past epoch"); + return; + } + + let boundary_height = epocher + .last(our_epoch) + .expect("our epoch should be covered by epoch strategy"); + debug!( + ?from, + %their_epoch, + %our_epoch, + %boundary_height, + "received backup message from future epoch, ensuring boundary finalization" + ); + self.marshal + .hint_finalized(boundary_height, NonEmptyVec::new(from)); + } + + /// Handle one finalized block delivered by marshal. + /// + /// Non-boundary blocks are acknowledged immediately. A boundary block must + /// carry the next epoch's public [`Payload::EpochInfo`]; once it does, the + /// actor stops the current Simplex engine and enters the next epoch using + /// that public peer set. + async fn handle_finalized( + &mut self, + epocher: &FixedEpocher, + active: &mut ActiveEpoch, + block: Arc, + acknowledgement: ACK, + channels: &mut Channels, + ) -> bool + where + S: Sender::PublicKey>, + R: Receiver::PublicKey>, + { + let height = block.height(); + if epocher.last(active.epoch) != Some(height) { + acknowledgement.acknowledge(); + return true; + } + + let next_epoch = active.epoch.next(); + let Some(Payload::EpochInfo(info)) = block.payload() else { + panic!("boundary block of epoch {} missing EpochInfo", active.epoch); + }; + if info.epoch != next_epoch { + panic!( + "boundary block of epoch {} carries epoch info for wrong epoch (got: {}, expected: {})", + active.epoch, info.epoch, next_epoch + ); + } + + let Some(boundary) = self.marshal.get_block(height).await else { + // Marshal cancels pending reads when it stops; only an + // intact-but-absent boundary block is a retention violation. + if self.context.stopped().now_or_never().is_some() { + debug!(%height, "boundary block read canceled during shutdown"); + return false; + } + panic!("missing finalized boundary block at height {height}"); + }; + let floor = Floor::Genesis(MV::commitment(&boundary)); + + let next = self + .enter_epoch( + next_epoch, + floor, + info.participants().tracked_peers(), + channels, + ) + .await; + let next = match next { + Ok(next) => next, + Err(EnterEpochError::GateClosed) => { + debug!(%next_epoch, "epoch gate closed before boundary transition"); + return false; + } + Err(EnterEpochError::MuxClosed) => { + debug!(%next_epoch, "consensus mux closed before boundary transition"); + return false; + } + Err(EnterEpochError::Stopped) => { + debug!(%next_epoch, "context shutdown while waiting to enter epoch"); + return false; + } + }; + + *active = next; + acknowledgement.acknowledge(); + true + } + + /// Enter an epoch and return the active engine handle. + /// + /// This is the only path that tracks consensus peers, opens epoch-scoped + /// mux subchannels, constructs the Simplex engine, and updates the current + /// epoch metric. Callers must abort the previous [`ActiveEpoch`] before + /// replacing it with the returned value. + async fn enter_epoch( + &mut self, + epoch: Epoch, + floor: Floor, + peers: impl Into::PublicKey>> + Send, + channels: &mut Channels, + ) -> Result + where + S: Sender::PublicKey>, + R: Receiver::PublicKey>, + { + // Shutdown is polled first so a stop signal wins over an + // already-marked gate. + let mut shutdown = self.context.stopped(); + select! { + _ = &mut shutdown => { + return Err(EnterEpochError::Stopped); + }, + result = self.gate.wait(epoch) => { + if result.is_err() { + return Err(EnterEpochError::GateClosed); + } + }, + }; + drop(shutdown); + + let peer_set_id = epoch + .get() + .checked_add(EPOCH_PEER_SET_OFFSET) + .expect("epoch peer-set ID overflow"); + let _ = self.manager.track(peer_set_id, peers); + let scheme = self + .provider + .scheme(epoch) + .unwrap_or_else(|| panic!("missing consensus scheme for epoch {epoch}")); + let context = self + .context + .child("consensus_engine") + .with_attribute("epoch", epoch); + let engine = simplex::Engine::new( + context, + simplex::Config { + scheme: scheme.as_ref().clone(), + elector: self.simplex.elector.clone(), + blocker: self.oracle.clone(), + automaton: self.application.clone(), + relay: self.application.clone(), + reporter: self.marshal.clone(), + strategy: self.strategy.clone(), + partition: format!("{}_consensus_{epoch}", self.partition_prefix), + mailbox_size: self.simplex.mailbox_size, + epoch, + floor, + replay_buffer: self.simplex.replay_buffer, + write_buffer: self.simplex.write_buffer, + page_cache: self.page_cache_ref.clone(), + leader_timeout: self.simplex.leader_timeout, + certification_timeout: self.simplex.certification_timeout, + timeout_retry: self.simplex.timeout_retry, + fetch_timeout: self.simplex.fetch_timeout, + fetch_concurrent: self.simplex.fetch_concurrent, + activity_timeout: self.simplex.activity_timeout, + skip_timeout: self.simplex.skip_timeout, + forwarding: self.simplex.forwarding, + }, + ); + + // Each epoch is registered exactly once, so a registration failure + // means the muxer has stopped: the vote and resolver muxers exit with + // this context, and the externally owned certificate mux may be + // stopped by its owner at any time. Both are clean-stop conditions. + let Ok(vote) = channels.vote.register(epoch.get()).await else { + return Err(EnterEpochError::MuxClosed); + }; + let Ok(certificate) = channels.certificate.register(epoch.get()).await else { + return Err(EnterEpochError::MuxClosed); + }; + let Ok(resolver) = channels.resolver.register(epoch.get()).await else { + return Err(EnterEpochError::MuxClosed); + }; + let handle = engine.start(vote, certificate, resolver); + let _ = self.latest_epoch.try_set(epoch.get()); + + info!(%epoch, "entered epoch"); + Ok(ActiveEpoch { epoch, handle }) + } +} diff --git a/glue/src/dkg/orchestrator/mailbox.rs b/glue/src/dkg/orchestrator/mailbox.rs new file mode 100644 index 00000000000..32230cb25c9 --- /dev/null +++ b/glue/src/dkg/orchestrator/mailbox.rs @@ -0,0 +1,73 @@ +//! Mailbox for the [`Actor`]. +//! +//! [`Actor`]: super::Actor + +use crate::dkg::ReshareBlock; +use commonware_actor::{ + Feedback, + mailbox::{Policy, Sender}, +}; +use commonware_consensus::{Reporter, marshal::Update}; +use commonware_utils::{Acknowledgement, acknowledgement::Exact}; +use std::{collections::VecDeque, sync::Arc}; + +/// Messages that can be sent to the orchestrator. +pub enum Message +where + B: ReshareBlock, + A: Acknowledgement, +{ + Finalized { block: Arc, acknowledgement: A }, +} + +impl Policy for Message +where + B: ReshareBlock, + A: Acknowledgement, +{ + type Overflow = VecDeque; + + fn handle(overflow: &mut VecDeque, message: Self) { + // Ensure delivery + overflow.push_back(message); + } +} + +/// Inbound communication channel for epoch transitions. +#[derive(Debug, Clone)] +pub struct Mailbox +where + B: ReshareBlock, + A: Acknowledgement, +{ + sender: Sender>, +} + +impl Mailbox +where + B: ReshareBlock, + A: Acknowledgement, +{ + /// Create a new [Mailbox]. + pub const fn new(sender: Sender>) -> Self { + Self { sender } + } +} + +impl Reporter for Mailbox +where + B: ReshareBlock, + A: Acknowledgement, +{ + type Activity = Update; + + fn report(&mut self, activity: Self::Activity) -> Feedback { + let Update::Block(block, acknowledgement) = activity else { + return Feedback::Ok; + }; + self.sender.enqueue(Message::Finalized { + block, + acknowledgement, + }) + } +} diff --git a/glue/src/dkg/orchestrator/mod.rs b/glue/src/dkg/orchestrator/mod.rs new file mode 100644 index 00000000000..c749d1cf7bc --- /dev/null +++ b/glue/src/dkg/orchestrator/mod.rs @@ -0,0 +1,899 @@ +//! Orchestrate [`Epoch`]-specific Simplex engines. +//! +//! The orchestrator is the bridge between finalized epoch material and +//! [`simplex`](commonware_consensus::simplex) consensus. It starts one Simplex +//! engine for the locally resolved epoch, watches marshal's finalized block +//! stream, and moves to the next epoch whenever the current epoch's final block +//! is finalized with the next [`EpochInfo`](crate::dkg::types::EpochInfo). +//! +//! # Epoch Lifecycle +//! +//! Epoch changes are driven by finalized blocks: +//! +//! 1. Startup resolves an epoch, peer set, and floor from marshal or state sync. +//! 2. The orchestrator tracks the peer set, loads the epoch scheme from its +//! [`Provider`](commonware_cryptography::certificate::Provider), opens +//! epoch-specific P2P subchannels, and starts Simplex. +//! 3. Marshal reports finalized blocks through [`Mailbox`]. +//! 4. When the finalized block is the final block of the active epoch, the +//! orchestrator extracts the next epoch's public `EpochInfo`, tracks the next +//! peer set, aborts the old Simplex engine, and starts the next one from the +//! boundary commitment. +//! +//! ```text +//! marshal boundary or state-sync artifact +//! | +//! v +//! Provider::scheme(epoch) +//! | +//! v +//! Simplex engine for epoch N +//! | +//! v +//! marshal finalized block stream +//! | +//! v +//! final block of epoch N carries EpochInfo(N + 1) +//! | +//! v +//! abort epoch N + start epoch N + 1 +//! ``` +//! +//! # Marshal Boundary +//! +//! Epoch zero is anchored by marshal's height-zero block. Later epochs are +//! anchored by the last finalized block of the previous epoch. Ordinary restart +//! expects that boundary block to remain in marshal's local finalized block +//! archive; see [`crate::dkg`] for the marshal retention requirement. +//! +//! # Catching Up +//! +//! Consensus votes are multiplexed by epoch. If the vote mux receives a message +//! for a future epoch that has not been registered locally, the node is behind. +//! The orchestrator hints marshal to fetch the boundary finalization needed to +//! reach that epoch, allowing normal marshal delivery to drive the transition. +//! +//! # Configuration +//! +//! [`Config`] wires together marshal, the application automaton/relay, the +//! scheme provider, P2P manager/blocker, optional state-sync material, network +//! channels, and persistence partitions. [`SimplexConfig`] contains the +//! per-epoch Simplex tunables; callers must provide these explicitly rather +//! than relying on hidden defaults. +//! +//! [`Epoch`]: commonware_consensus::types::Epoch + +mod mailbox; +pub use mailbox::{Mailbox, Message}; + +mod actor; +pub use actor::{Actor, Config, SimplexConfig, StateSync}; + +#[cfg(test)] +mod tests { + use super::{Actor, Config, StateSync}; + use crate::dkg::{ + anchor::Artifact, + fence::Fence, + tests::mocks, + types::{EpochInfo, EpochOutcome, Payload}, + }; + use commonware_actor::Feedback; + use commonware_consensus::{ + Heightable, Reporter, Reporters, + marshal::{self, Start as MarshalStart, resolver::p2p as marshal_resolver}, + simplex::types::{Activity, Finalization, Finalize, Proposal}, + types::{Epoch, FixedEpocher, Height, Round, View, ViewDelta}, + }; + use commonware_cryptography::{ + Digestible as _, + bls12381::{dkg::feldman_desmedt::deal, primitives::sharing::Mode}, + certificate::Verifier as _, + sha256::Sha256, + }; + use commonware_macros::select; + use commonware_p2p::{ + simulated::{Config as NetworkConfig, Link, Network, Oracle}, + utils::mux::{Builder, Muxer}, + }; + use commonware_parallel::Sequential; + use commonware_runtime::{ + Clock as _, Handle, Quota, Runner, Spawner as _, Supervisor as _, buffer::paged::CacheRef, + deterministic, + }; + use commonware_storage::archive::immutable; + use commonware_utils::{ + N3f1, NZU16, NZU32, NZU64, NZUsize, TestRng, acknowledgement::Exact, ordered::Set, + }; + use std::time::Duration; + + const BACKFILL_CHANNEL: u64 = 0; + const VOTE_CHANNEL: u64 = 1; + const CERTIFICATE_CHANNEL: u64 = 2; + const RESOLVER_CHANNEL: u64 = 3; + const TEST_QUOTA: Quota = Quota::per_second(NZU32!(1_000_000)); + const LINK: Link = Link { + latency: Duration::from_millis(1), + jitter: Duration::ZERO, + success_rate: 1.0, + }; + type TestStateSync = StateSync; + + struct Cluster { + nodes: Vec, + boundary: mocks::TestBlock, + oracle: Oracle, + network_handle: Handle<()>, + } + + impl Cluster { + async fn start(context: &mut deterministic::Context, nodes: usize) -> Self { + Self::start_with_seeded_first(context, nodes, true).await + } + + async fn start_with_seeded_first( + context: &mut deterministic::Context, + nodes: usize, + seed_first: bool, + ) -> Self { + let fixture = mocks::scheme_fixture_n(context, nodes as u32); + Self::start_with_fixture(context, &fixture, seed_first).await + } + + async fn start_with_fixture( + context: &mut deterministic::Context, + fixture: &mocks::SchemeFixture, + seed_first: bool, + ) -> Self { + Self::start_with_fixture_and_state_sync(context, fixture, seed_first, None).await + } + + async fn start_with_state_sync( + context: &mut deterministic::Context, + fixture: &mocks::SchemeFixture, + state_sync: TestStateSync, + ) -> Self { + Self::start_with_fixture_and_state_sync(context, fixture, false, Some(state_sync)).await + } + + async fn start_with_fixture_and_state_sync( + context: &mut deterministic::Context, + fixture: &mocks::SchemeFixture, + seed_first: bool, + first_state_sync: Option, + ) -> Self { + Self::start_with_fixture_state_sync_and_gate_epoch( + context, + fixture, + seed_first, + first_state_sync, + Epoch::new(1), + ) + .await + } + + async fn start_with_gate_epoch( + context: &mut deterministic::Context, + fixture: &mocks::SchemeFixture, + seed_first: bool, + gate_epoch: Epoch, + ) -> Self { + Self::start_with_fixture_state_sync_and_gate_epoch( + context, fixture, seed_first, None, gate_epoch, + ) + .await + } + + async fn start_with_fixture_state_sync_and_gate_epoch( + context: &mut deterministic::Context, + fixture: &mocks::SchemeFixture, + seed_first: bool, + mut first_state_sync: Option, + gate_epoch: Epoch, + ) -> Self { + let participants = fixture.participants.clone(); + let boundary = make_height_one_block(participants[0].clone(), &participants); + let nodes = participants.len(); + + let (network, oracle) = Network::new_with_peers( + context.child("network"), + NetworkConfig { + max_size: 1024 * 1024, + disconnect_on_block: true, + tracked_peer_sets: NZUsize!(1), + }, + participants.clone(), + ) + .await; + let network_handle = network.start(); + for from in &participants { + for to in &participants { + if from != to { + oracle + .add_link(from.clone(), to.clone(), LINK) + .await + .expect("failed to add link"); + } + } + } + + let mut started = Vec::with_capacity(nodes); + for index in 0..nodes { + let boundary = (seed_first || index > 0).then(|| boundary.clone()); + let state_sync = if index == 0 { + first_state_sync.take() + } else { + None + }; + started.push( + Node::start_with_gate_epoch( + context.child("node").with_attribute("index", index), + &oracle, + fixture, + index, + boundary, + state_sync, + gate_epoch, + ) + .await, + ); + } + + Self { + nodes: started, + boundary, + oracle, + network_handle, + } + } + + async fn restart( + &mut self, + context: deterministic::Context, + fixture: &mocks::SchemeFixture, + index: usize, + ) { + self.nodes[index].abort(); + self.nodes[index] = + Node::start(context, &self.oracle, fixture, index, None, None).await; + } + } + + impl Drop for Cluster { + fn drop(&mut self) { + for node in &mut self.nodes { + node.abort(); + } + self.network_handle.abort(); + } + } + + struct Node { + marshal: mocks::TestMarshalMailbox, + application: mocks::MockApplication, + fence: Fence, + orchestrator_handle: Handle<()>, + certificate_mux_handle: Handle>, + marshal_handle: Handle<()>, + } + + impl Node { + async fn start( + context: deterministic::Context, + oracle: &Oracle, + fixture: &mocks::SchemeFixture, + index: usize, + boundary: Option, + state_sync: Option, + ) -> Self { + Self::start_with_gate_epoch( + context, + oracle, + fixture, + index, + boundary, + state_sync, + Epoch::new(1), + ) + .await + } + + async fn start_with_gate_epoch( + context: deterministic::Context, + oracle: &Oracle, + fixture: &mocks::SchemeFixture, + index: usize, + boundary: Option, + state_sync: Option, + gate_epoch: Epoch, + ) -> Self { + let public_key = fixture.participants[index].clone(); + let control = oracle.control(public_key.clone()); + let page_cache = CacheRef::from_pooler(&context, NZU16!(1024), NZUsize!(16)); + let partition_prefix = format!("orchestrator-node-{index}"); + + let backfill = control + .register(BACKFILL_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register marshal backfill channel"); + let resolver = marshal_resolver::init( + context.child("marshal_resolver"), + marshal_resolver::Config { + public_key: public_key.clone(), + peer_provider: oracle.manager(), + blocker: control.clone(), + mailbox_size: NZUsize!(16), + initial: Duration::from_millis(100), + timeout: Duration::from_millis(200), + fetch_retry_timeout: Duration::from_millis(100), + priority_requests: false, + priority_responses: false, + }, + backfill, + ); + + let finalizations_by_height = + immutable::Archive::init(context.child("finalizations_by_height"), { + let _: () = mocks::TestScheme::certificate_codec_config_unbounded(); + archive_config( + &partition_prefix, + "finalizations_by_height", + page_cache.clone(), + (), + ) + }) + .await + .expect("failed to initialize finalizations archive"); + let finalized_blocks = immutable::Archive::init( + context.child("finalized_blocks"), + archive_config(&partition_prefix, "finalized_blocks", page_cache, ()), + ) + .await + .expect("failed to initialize finalized blocks archive"); + + let genesis = + make_genesis_block(public_key.clone(), fixture.participants.iter().cloned()); + let (marshal_actor, mut marshal, _) = marshal::core::Actor::init( + context.child("marshal"), + finalizations_by_height, + finalized_blocks, + marshal::Config { + provider: mocks::TestProvider::new(fixture.schemes[index].clone()), + epocher: FixedEpocher::new(NZU64!(2)), + start: MarshalStart::Genesis(genesis), + partition_prefix: partition_prefix.clone(), + mailbox_size: NZUsize!(16), + view_retention_timeout: ViewDelta::new(8), + prunable_items_per_section: NZU64!(10), + page_cache: CacheRef::from_pooler(&context, NZU16!(1024), NZUsize!(16)), + replay_buffer: NZUsize!(1024), + key_write_buffer: NZUsize!(1024), + value_write_buffer: NZUsize!(1024), + block_codec_config: (), + max_repair: NZUsize!(4), + max_pending_acks: NZUsize!(4), + strategy: Sequential, + }, + ) + .await; + let application = mocks::MockApplication::default(); + let (fence, gate) = Fence::new(gate_epoch); + let (actor, mailbox) = Actor::new( + context.child("orchestrator"), + Config { + oracle: control.clone(), + manager: oracle.manager(), + provider: mocks::TestProvider::new(fixture.schemes[index].clone()), + marshal: marshal.clone(), + application: application.clone(), + strategy: Sequential, + simplex: mocks::simplex_config(), + gate, + state_sync, + blocks_per_epoch: NZU64!(2), + muxer_size: 16, + mailbox_size: NZUsize!(16), + partition_prefix, + }, + ); + let reporters = Reporters::from((mocks::MarshalApplication::default(), mailbox)); + let marshal_handle = marshal_actor.start_unbuffered(reporters, resolver); + + if let Some(block) = &boundary { + assert!( + marshal + .certified(block.context().round, block.clone()) + .await + ); + let finalization = make_finalization( + Proposal::new( + Round::new(Epoch::zero(), View::new(1)), + View::zero(), + block.digest(), + ), + &fixture.schemes, + ); + assert_eq!( + marshal.report(Activity::Finalization(finalization)), + Feedback::Ok + ); + } + + let votes = control + .register(VOTE_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register vote channel"); + let certificates = control + .register(CERTIFICATE_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register certificate channel"); + let (certificate_mux, certificates) = Muxer::builder( + context.child("certificate_mux"), + certificates.0, + certificates.1, + 16, + ) + .build(); + let certificate_mux_handle = certificate_mux.start(); + let simplex_resolver = control + .register(RESOLVER_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register simplex resolver channel"); + let orchestrator_handle = actor.start(votes, certificates, simplex_resolver); + + Self { + marshal, + application, + fence, + orchestrator_handle, + certificate_mux_handle, + marshal_handle, + } + } + + fn abort(&mut self) { + self.orchestrator_handle.abort(); + self.certificate_mux_handle.abort(); + self.marshal_handle.abort(); + } + } + + fn archive_config( + prefix: &str, + name: &str, + page_cache: CacheRef, + codec_config: C, + ) -> immutable::Config { + immutable::Config { + metadata_partition: format!("{prefix}-{name}-metadata"), + freezer_table_partition: format!("{prefix}-{name}-freezer-table"), + freezer_table_initial_size: 64, + freezer_table_resize_frequency: 10, + freezer_table_resize_chunk_size: 10, + freezer_key_partition: format!("{prefix}-{name}-freezer-key"), + freezer_key_page_cache: page_cache, + freezer_value_partition: format!("{prefix}-{name}-freezer-value"), + freezer_value_target_size: 1024, + freezer_value_compression: None, + ordinal_partition: format!("{prefix}-{name}-ordinal"), + items_per_section: NZU64!(10), + codec_config, + replay_buffer: NZUsize!(1024), + freezer_key_write_buffer: NZUsize!(1024), + freezer_value_write_buffer: NZUsize!(1024), + ordinal_write_buffer: NZUsize!(1024), + } + } + + async fn wait_for_block( + context: &deterministic::Context, + marshal: &mocks::TestMarshalMailbox, + height: Height, + ) -> mocks::TestBlock { + for _ in 0..50 { + if let Some(block) = marshal.get_block(height).await { + return block; + } + context.sleep(Duration::from_millis(10)).await; + } + panic!("missing finalized block at height {height}"); + } + + async fn wait_for_proposal( + context: &deterministic::Context, + nodes: &[Node], + epoch: Epoch, + ) -> mocks::TestContext { + for _ in 0..50 { + for node in nodes { + if let Some(proposal) = node + .application + .proposals() + .into_iter() + .find(|proposal| proposal.round.epoch() == epoch) + { + return proposal; + } + } + context.sleep(Duration::from_millis(10)).await; + } + panic!("missing application proposal"); + } + + fn make_height_one_block( + leader: mocks::TestPublicKey, + participants: &[mocks::TestPublicKey], + ) -> mocks::TestBlock { + let genesis = make_genesis_block(leader.clone(), participants.iter().cloned()); + let context = mocks::TestContext { + round: Round::new(Epoch::zero(), View::new(1)), + leader, + parent: (View::zero(), genesis.digest()), + }; + let info = make_epoch_info(Epoch::new(1), participants.iter().cloned()); + mocks::TestBlock::new::(context, genesis.digest(), Height::new(1), 1) + .with_payload::( + NZU32!(16), + Payload::EpochInfo(info), + ) + } + + fn make_genesis_block( + leader: mocks::TestPublicKey, + participants: impl IntoIterator, + ) -> mocks::TestBlock { + let info = make_epoch_info(Epoch::zero(), participants); + mocks::genesis_block(leader) + .with_payload::( + NZU32!(16), + Payload::EpochInfo(info), + ) + } + + fn make_epoch_info( + epoch: Epoch, + participants: impl IntoIterator, + ) -> EpochInfo { + let participants = Set::from_iter_dedup(participants); + let (output, _) = deal::( + TestRng::new(epoch.get() + 1), + Mode::NonZeroCounter, + participants.clone(), + ) + .expect("failed to create test DKG output"); + EpochInfo { + outcome: EpochOutcome::Success, + epoch, + output, + players: participants.clone(), + next_players: participants, + } + } + + fn make_finalization( + proposal: Proposal, + schemes: &[mocks::TestScheme], + ) -> Finalization { + let finalizes = schemes + .iter() + .map(|scheme| Finalize::sign(scheme, proposal.clone()).unwrap()) + .collect::>(); + Finalization::from_finalizes(&schemes[0], &finalizes, &Sequential) + .expect("finalization quorum") + } + + #[test] + fn cluster_serves_genesis_through_marshal() { + let runner = deterministic::Runner::default(); + runner.start(|mut context| async move { + let cluster = Cluster::start_with_seeded_first(&mut context, 1, false).await; + let block = cluster.nodes[0] + .marshal + .get_block(Height::zero()) + .await + .expect("genesis should be available through marshal"); + + assert_eq!(block.height(), Height::zero()); + }); + } + + #[test] + fn initial_epoch_starts_without_mailbox_transition() { + let runner = deterministic::Runner::default(); + runner.start(|mut context| async move { + let cluster = Cluster::start_with_seeded_first(&mut context, 4, false).await; + let proposal = wait_for_proposal(&context, &cluster.nodes, Epoch::zero()).await; + + assert_eq!(proposal.round.epoch(), Epoch::zero()); + }); + } + + #[test] + fn active_engine_channel_close_stops_orchestrator() { + let runner = deterministic::Runner::timed(Duration::from_secs(10)); + runner.start(|mut context| async move { + let mut cluster = Cluster::start_with_seeded_first(&mut context, 1, false).await; + let proposal = wait_for_proposal(&context, &cluster.nodes, Epoch::zero()).await; + assert_eq!(proposal.round.epoch(), Epoch::zero()); + + let node = &mut cluster.nodes[0]; + node.certificate_mux_handle.abort(); + select! { + result = &mut node.orchestrator_handle => { + result.expect("orchestrator should stop cleanly"); + }, + _ = context.sleep(Duration::from_secs(1)) => { + panic!("orchestrator stayed alive after active engine stopped"); + }, + }; + }); + } + + #[test] + fn certificate_mux_close_during_boundary_transition_stops_cleanly() { + let runner = deterministic::Runner::timed(Duration::from_secs(10)); + runner.start(|mut context| async move { + let fixture = mocks::scheme_fixture_n(&mut context, 1); + let mut cluster = + Cluster::start_with_gate_epoch(&mut context, &fixture, true, Epoch::zero()).await; + let proposal = wait_for_proposal(&context, &cluster.nodes, Epoch::zero()).await; + assert_eq!(proposal.round.epoch(), Epoch::zero()); + + // The seeded boundary block parks the orchestrator inside + // enter_epoch(1) on the closed gate. Stop the externally owned + // certificate mux before releasing the gate so the transition + // observes a closed mux when it registers epoch subchannels. + // Marshal is stopped alongside it, as during node shutdown: + // exiting the transition drops the boundary acknowledgement, which + // a still-running marshal intentionally treats as fatal. + let node = &mut cluster.nodes[0]; + node.certificate_mux_handle.abort(); + node.marshal_handle.abort(); + context.sleep(Duration::from_millis(10)).await; + node.fence.mark(Epoch::new(1)); + + select! { + result = &mut node.orchestrator_handle => { + result.expect("orchestrator should stop cleanly"); + }, + _ = context.sleep(Duration::from_secs(1)) => { + panic!("orchestrator stayed alive after certificate mux closed"); + }, + }; + }); + } + + #[test] + fn shutdown_interrupts_boundary_gate_wait() { + let runner = deterministic::Runner::timed(Duration::from_secs(10)); + runner.start(|mut context| async move { + let fixture = mocks::scheme_fixture_n(&mut context, 1); + let cluster = + Cluster::start_with_gate_epoch(&mut context, &fixture, true, Epoch::zero()).await; + let proposal = wait_for_proposal(&context, &cluster.nodes, Epoch::zero()).await; + assert_eq!(proposal.round.epoch(), Epoch::zero()); + + context.sleep(Duration::from_millis(10)).await; + + context + .child("shutdown") + .stop(7, Some(Duration::from_secs(1))) + .await + .expect("shutdown should interrupt the boundary gate wait"); + }); + } + + #[test] + fn marshal_shutdown_during_startup_resolution_stops_cleanly() { + let runner = deterministic::Runner::timed(Duration::from_secs(10)); + runner.start(|mut context| async move { + let fixture = mocks::scheme_fixture_n(&mut context, 1); + let participants = fixture.participants.clone(); + let (network, oracle) = Network::new_with_peers( + context.child("network"), + NetworkConfig { + max_size: 1024 * 1024, + disconnect_on_block: true, + tracked_peer_sets: NZUsize!(1), + }, + participants.clone(), + ) + .await; + network.start(); + + let public_key = participants[0].clone(); + let control = oracle.control(public_key.clone()); + let page_cache = CacheRef::from_pooler(&context, NZU16!(1024), NZUsize!(16)); + let partition_prefix = "orchestrator-marshal-shutdown".to_string(); + + let finalizations_by_height = + immutable::Archive::init(context.child("finalizations_by_height"), { + let _: () = mocks::TestScheme::certificate_codec_config_unbounded(); + archive_config( + &partition_prefix, + "finalizations_by_height", + page_cache.clone(), + (), + ) + }) + .await + .expect("failed to initialize finalizations archive"); + let finalized_blocks = immutable::Archive::init( + context.child("finalized_blocks"), + archive_config(&partition_prefix, "finalized_blocks", page_cache, ()), + ) + .await + .expect("failed to initialize finalized blocks archive"); + let genesis = make_genesis_block(public_key.clone(), participants.iter().cloned()); + let (marshal_actor, marshal, _): (_, mocks::TestMarshalMailbox, _) = + marshal::core::Actor::<_, _, _, _, _, _, _, Exact>::init( + context.child("marshal"), + finalizations_by_height, + finalized_blocks, + marshal::Config { + provider: mocks::TestProvider::new(fixture.schemes[0].clone()), + epocher: FixedEpocher::new(NZU64!(2)), + start: MarshalStart::Genesis(genesis), + partition_prefix: partition_prefix.clone(), + mailbox_size: NZUsize!(16), + view_retention_timeout: ViewDelta::new(8), + prunable_items_per_section: NZU64!(10), + page_cache: CacheRef::from_pooler(&context, NZU16!(1024), NZUsize!(16)), + replay_buffer: NZUsize!(1024), + key_write_buffer: NZUsize!(1024), + value_write_buffer: NZUsize!(1024), + block_codec_config: (), + max_repair: NZUsize!(4), + max_pending_acks: NZUsize!(4), + strategy: Sequential, + }, + ) + .await; + + let (_fence, gate) = Fence::new(Epoch::new(1)); + let (actor, _mailbox): (_, super::Mailbox) = Actor::new( + context.child("orchestrator"), + Config { + oracle: control.clone(), + manager: oracle.manager(), + provider: mocks::TestProvider::new(fixture.schemes[0].clone()), + marshal: marshal.clone(), + application: mocks::MockApplication::default(), + strategy: Sequential, + simplex: mocks::simplex_config(), + gate, + state_sync: None, + blocks_per_epoch: NZU64!(2), + muxer_size: 16, + mailbox_size: NZUsize!(16), + partition_prefix, + }, + ); + let votes = control + .register(VOTE_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register vote channel"); + let certificates = control + .register(CERTIFICATE_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register certificate channel"); + let (certificate_mux, certificates) = Muxer::builder( + context.child("certificate_mux"), + certificates.0, + certificates.1, + 16, + ) + .build(); + certificate_mux.start(); + let simplex_resolver = control + .register(RESOLVER_CHANNEL, TEST_QUOTA) + .await + .expect("failed to register simplex resolver channel"); + + // The marshal actor is never started, so startup resolution parks + // on an unserved processed-height read. + let mut orchestrator_handle = actor.start(votes, certificates, simplex_resolver); + context.sleep(Duration::from_millis(10)).await; + + // Signal shutdown, then drop the marshal actor: its mailbox + // cancels the pending reads only after the stop signal is visible, + // mirroring marshal winning the shutdown race. + let stopper = context.child("stopper"); + context.child("stop").spawn(|_| async move { + let _ = stopper.stop(0, None).await; + }); + context.sleep(Duration::from_millis(10)).await; + drop(marshal_actor); + + select! { + result = &mut orchestrator_handle => { + result.expect("orchestrator should stop cleanly"); + }, + _ = context.sleep(Duration::from_secs(1)) => { + panic!("orchestrator stayed alive after marshal shutdown"); + }, + }; + }); + } + + #[test] + #[should_panic(expected = "state sync artifact and floor must be in the same epoch")] + fn state_sync_rejects_mismatched_floor_epoch() { + let runner = deterministic::Runner::default(); + runner.start(|mut context| async move { + let fixture = mocks::scheme_fixture_n(&mut context, 4); + let artifact = Artifact { + epoch: Epoch::zero(), + finalization: None, + info: make_epoch_info(Epoch::zero(), fixture.participants.iter().cloned()), + }; + let genesis = make_genesis_block( + fixture.participants[0].clone(), + fixture.participants.iter().cloned(), + ); + let floor = make_finalization( + Proposal::new( + Round::new(Epoch::new(1), View::new(1)), + View::zero(), + genesis.digest(), + ), + &fixture.schemes, + ); + + let state_sync = StateSync { artifact, floor }; + let _cluster = Cluster::start_with_state_sync(&mut context, &fixture, state_sync).await; + }); + } + + #[test] + fn future_epoch_vote_hints_marshal_to_fetch_boundary_finalization() { + let runner = deterministic::Runner::default(); + runner.start(|mut context| async move { + let cluster = Cluster::start_with_seeded_first(&mut context, 4, false).await; + let caught_up = + wait_for_block(&context, &cluster.nodes[0].marshal, Height::new(1)).await; + + assert_eq!(caught_up.digest(), cluster.boundary.digest()); + }); + } + + #[test] + fn finalized_boundary_enters_next_epoch() { + let runner = deterministic::Runner::default(); + runner.start(|mut context| async move { + let cluster = Cluster::start(&mut context, 4).await; + let stored = wait_for_block(&context, &cluster.nodes[0].marshal, Height::new(1)).await; + + let proposal = wait_for_proposal(&context, &cluster.nodes, Epoch::new(1)).await; + + assert_eq!(stored.digest(), cluster.boundary.digest()); + assert_eq!(proposal.round.epoch(), Epoch::new(1)); + assert_eq!(proposal.parent.1, cluster.boundary.digest()); + }); + } + + #[test] + fn recovered_node_starts_from_processed_epoch() { + let runner = deterministic::Runner::default(); + runner.start(|mut context| async move { + let fixture = mocks::scheme_fixture_n(&mut context, 4); + let mut cluster = Cluster::start_with_fixture(&mut context, &fixture, true).await; + let stored = wait_for_block(&context, &cluster.nodes[0].marshal, Height::new(1)).await; + assert_eq!(stored.digest(), cluster.boundary.digest()); + + cluster + .restart( + context + .child("node") + .with_attribute("index", 0) + .with_attribute("restart", 1), + &fixture, + 0, + ) + .await; + let proposal = wait_for_proposal(&context, &cluster.nodes[0..1], Epoch::new(1)).await; + + assert_eq!(proposal.round.epoch(), Epoch::new(1)); + }); + } +} diff --git a/glue/src/dkg/reshare/actor/dealing.rs b/glue/src/dkg/reshare/actor/dealing.rs new file mode 100644 index 00000000000..ff477a91d06 --- /dev/null +++ b/glue/src/dkg/reshare/actor/dealing.rs @@ -0,0 +1,518 @@ +use crate::dkg::{ + ParticipantsProvider, Registrar, ReshareBlock, SecretStore, + reshare::{ + Actor, EpochInfoResponse, Message as MailboxMessage, + metrics::Phase, + store::{Dealer, Player, Store}, + }, + types::Message, +}; +use commonware_codec::{Decode, Encode}; +use commonware_consensus::{ + marshal::core::Variant as MarshalVariant, + types::{Epoch, EpochPhase, Epocher}, +}; +use commonware_cryptography::{ + BatchVerifier, Signer, + bls12381::{dkg::feldman_desmedt::Verdict, primitives::variant::Variant as BlsVariant}, + certificate::Scheme, +}; +use commonware_macros::select_loop; +use commonware_p2p::{Blocker, Manager, Message as NetworkMessage, Receiver, Recipients, Sender}; +use commonware_parallel::Strategy; +use commonware_runtime::{ + BufferPooler, Clock, Metrics, Spawner, Storage, telemetry::traces::TracedExt as _, +}; +use commonware_utils::{Acknowledgement, channel::fallible::OneshotExt}; +use rand_core::CryptoRng; +use std::ops::ControlFlow; +use tracing::{Instrument as _, debug, info, info_span, warn}; + +impl Actor +where + E: Spawner + CryptoRng + Metrics + BufferPooler + Clock + Storage, + B: ReshareBlock, + V: BlsVariant, + C: Signer, + M: Manager, + X: Blocker, + P: ParticipantsProvider, + SS: SecretStore, + T: Strategy, + BV: BatchVerifier + Send + 'static, + S: Scheme, + MV: MarshalVariant, + R: Registrar, + A: Acknowledgement, +{ + /// Run the early dealing phase for `epoch`. + /// + /// The phase processes inbound dealer messages and acknowledgements while + /// finalized blocks remain in [`EpochPhase::Early`]. It returns after the + /// final early block is acknowledged. + pub(super) async fn dealing( + &mut self, + epoch: Epoch, + store: &mut Store, + mut dealer: Option<&mut Dealer>, + mut player: Option<&mut Player>, + (mut sender, mut receiver): (SE, RE), + ) -> ControlFlow<()> + where + SE: Sender, + RE: Receiver, + { + self.metrics.set_phase(Phase::Dealing); + + select_loop! { + self.context, + on_stopped => { + debug!("shutdown signal received"); + return ControlFlow::Break(()); + }, + Some(message) = self.mailbox.recv() else { + debug!("mailbox closed, shutting down"); + return ControlFlow::Break(()); + } => match message { + MailboxMessage::NextLog { span, response, .. } => { + let process = info_span!(parent: &span, "dkg.reshare.actor.dealing.next_log"); + process.in_scope(|| { + let _ = response.send_lossy(None); + }); + } + MailboxMessage::ReleaseLog { .. } => {} + MailboxMessage::EpochInfo { span, response, .. } => { + let process = info_span!(parent: &span, "dkg.reshare.actor.dealing.epoch_info"); + process.in_scope(|| { + let _ = response.send_lossy(EpochInfoResponse::Pending); + }); + } + MailboxMessage::Finalized { + span, + block, + response, + } => { + let process = info_span!( + parent: &span, + "dkg.reshare.actor.dealing.finalized", + height = block.height().traced() + ); + let done = async { + let bounds = self + .epocher + .containing(block.height()) + .expect("epocher must know of block height"); + assert_eq!(bounds.epoch(), epoch, "dealing received future epoch block"); + assert_eq!( + bounds.phase(), + EpochPhase::Early, + "dealing received block after early phase" + ); + + if let Some(dealer) = dealer.as_deref_mut() { + Self::send_dealings( + &self.signer.public_key(), + store, + epoch, + dealer, + player.as_deref_mut(), + &mut sender, + ) + .await; + } + + let done = self + .epocher + .midpoint(epoch) + .and_then(|midpoint| midpoint.previous()) + == Some(block.height()); + response.acknowledge(); + done + } + .instrument(process) + .await; + if done { + return ControlFlow::Continue(()); + } + } + }, + Ok(message) = receiver.recv() else { + debug!("dealing channel closed, shutting down"); + return ControlFlow::Break(()); + } => { + self.handle_message( + epoch, + store, + dealer.as_deref_mut(), + player.as_deref_mut(), + &mut sender, + message, + ) + .await + }, + }; + + ControlFlow::Break(()) + } + + async fn handle_message( + &mut self, + epoch: Epoch, + store: &mut Store, + dealer: Option<&mut Dealer>, + player: Option<&mut Player>, + sender: &mut SE, + (from, bytes): NetworkMessage, + ) where + SE: Sender, + { + let message = + match Message::::decode_cfg(bytes.as_ref(), &self.max_participants) { + Ok(message) => message, + Err(error) => { + commonware_p2p::block!( + self.blocker, + from, + ?epoch, + ?error, + "failed to decode dealing message" + ); + return; + } + }; + + match message { + Message::Dealer(public, private) => { + let Some(player) = player else { + commonware_p2p::block!( + self.blocker, + from, + ?epoch, + "dealing sent to non-player" + ); + return; + }; + let ack = match player + .handle(store, epoch, from.clone(), public, private) + .await + { + Verdict::Valid(ack) => ack, + Verdict::Skip => return, + Verdict::Fault => { + commonware_p2p::block!(self.blocker, from, ?epoch, "invalid dealing"); + return; + } + }; + + self.metrics.record_share(&from, epoch.get()); + info!(?epoch, dealer = ?from, "received dealing"); + let sent = sender.send( + Recipients::One(from.clone()), + Message::::Ack(ack).encode(), + true, + ); + if sent.is_empty() { + warn!(?epoch, dealer = ?from, "failed to send ack"); + } + } + Message::Ack(ack) => { + let Some(dealer) = dealer else { + commonware_p2p::block!(self.blocker, from, ?epoch, "ack sent to non-dealer"); + return; + }; + match dealer.handle(store, epoch, from.clone(), ack).await { + Verdict::Valid(()) => { + self.metrics.record_ack(&from, epoch.get()); + info!(?epoch, player = ?from, "received ack"); + } + Verdict::Skip => {} + Verdict::Fault => { + commonware_p2p::block!(self.blocker, from, ?epoch, "invalid ack signature"); + } + } + } + } + } + + async fn send_dealings( + public_key: &C::PublicKey, + store: &mut Store, + epoch: Epoch, + dealer: &mut Dealer, + mut player: Option<&mut Player>, + sender: &mut SE, + ) where + SE: Sender, + { + for (recipient, public, private) in dealer.shares_to_distribute().collect::>() { + if recipient == *public_key { + let Some(player) = player.as_deref_mut() else { + continue; + }; + let Verdict::Valid(ack) = player + .handle(store, epoch, public_key.clone(), public, private) + .await + else { + continue; + }; + let _ = dealer.handle(store, epoch, public_key.clone(), ack).await; + continue; + } + + let sent = sender.send( + Recipients::One(recipient.clone()), + Message::::Dealer(public, private).encode(), + true, + ); + if sent.is_empty() { + debug!(?epoch, ?recipient, "failed to send share"); + } else { + debug!(?epoch, ?recipient, "sent share"); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dkg::{ + fence::Fence, + reshare::actor::Config, + tests::mocks::{self, MemorySecretStore}, + }; + use commonware_actor::Feedback; + use commonware_consensus::{ + Reporter, + marshal::{self, Start as MarshalStart, core::Actor as MarshalActor}, + types::{FixedEpocher, ViewDelta}, + }; + use commonware_cryptography::{ + bls12381::primitives::sharing::Mode, certificate::Verifier as _, ed25519, + }; + use commonware_p2p::{ + Receiver, + simulated::{Config as NetworkConfig, Network}, + utils::mocks::inert_channel, + }; + use commonware_parallel::Sequential; + use commonware_runtime::{ + IoBuf, Runner, Supervisor as _, buffer::paged::CacheRef, deterministic, + }; + use commonware_storage::archive::immutable; + use commonware_utils::{ + Acknowledgement, NZU16, NZU32, NZU64, NZUsize, acknowledgement::Exact, ordered::Set, + }; + use std::{ + collections::VecDeque, + convert::Infallible, + marker::PhantomData, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; + + const TEST_NAMESPACE: &[u8] = b"_COMMONWARE_GLUE_DKG_RESHARE_DEALING_TEST"; + + type TestActor = Actor< + deterministic::Context, + mocks::TestBlock, + mocks::TestBlsVariant, + mocks::TestSigner, + mocks::TestManager, + mocks::TestBlocker, + StaticParticipants, + MemorySecretStore, + Sequential, + ed25519::Batch, + mocks::TestScheme, + mocks::TestMarshalVariant, + mocks::MockConsumer, + >; + + #[derive(Clone)] + struct StaticParticipants(Set); + + impl ParticipantsProvider for StaticParticipants { + type PublicKey = mocks::TestPublicKey; + + async fn participants(&mut self, _epoch: Epoch) -> Set { + self.0.clone() + } + } + + #[derive(Debug)] + struct QueuedReceiver { + peer: mocks::TestPublicKey, + messages: VecDeque, + received: Arc, + } + + impl Receiver for QueuedReceiver { + type Error = Infallible; + type PublicKey = mocks::TestPublicKey; + + async fn recv(&mut self) -> Result, Self::Error> { + let Some(message) = self.messages.pop_front() else { + futures::future::pending().await + }; + self.received.fetch_add(1, Ordering::SeqCst); + Ok((self.peer.clone(), message)) + } + } + + async fn marshal_mailbox( + context: deterministic::Context, + signer: &mocks::TestSigner, + scheme: mocks::TestScheme, + ) -> mocks::TestMarshalMailbox { + let page_cache = CacheRef::from_pooler(&context, NZU16!(1024), NZUsize!(8)); + let finalizations_by_height = + immutable::Archive::init(context.child("finalizations_by_height"), { + let _: () = mocks::TestScheme::certificate_codec_config_unbounded(); + archive_config("dealing-priority", "finalizations", page_cache.clone(), ()) + }) + .await + .expect("finalizations archive"); + let finalized_blocks = immutable::Archive::init( + context.child("finalized_blocks"), + archive_config("dealing-priority", "blocks", page_cache.clone(), ()), + ) + .await + .expect("blocks archive"); + + let (_actor, mailbox, _) = MarshalActor::<_, _, _, _, _, _, _, Exact>::init( + context.child("marshal"), + finalizations_by_height, + finalized_blocks, + marshal::Config { + provider: mocks::TestProvider::new(scheme), + epocher: FixedEpocher::new(NZU64!(2)), + start: MarshalStart::Genesis(mocks::genesis_block(signer.public_key())), + partition_prefix: "dealing-priority-marshal".into(), + mailbox_size: NZUsize!(16), + view_retention_timeout: ViewDelta::new(8), + prunable_items_per_section: NZU64!(10), + page_cache, + replay_buffer: NZUsize!(1024), + key_write_buffer: NZUsize!(1024), + value_write_buffer: NZUsize!(1024), + block_codec_config: (), + max_repair: NZUsize!(4), + max_pending_acks: NZUsize!(4), + strategy: Sequential, + }, + ) + .await; + mailbox + } + + fn archive_config( + prefix: &str, + name: &str, + page_cache: CacheRef, + codec_config: C, + ) -> immutable::Config { + immutable::Config { + metadata_partition: format!("{prefix}-{name}-metadata"), + freezer_table_partition: format!("{prefix}-{name}-freezer-table"), + freezer_table_initial_size: 64, + freezer_table_resize_frequency: 10, + freezer_table_resize_chunk_size: 10, + freezer_key_partition: format!("{prefix}-{name}-freezer-key"), + freezer_key_page_cache: page_cache, + freezer_value_partition: format!("{prefix}-{name}-freezer-value"), + freezer_value_target_size: 1024, + freezer_value_compression: None, + ordinal_partition: format!("{prefix}-{name}-ordinal"), + items_per_section: NZU64!(10), + codec_config, + replay_buffer: NZUsize!(1024), + freezer_key_write_buffer: NZUsize!(1024), + freezer_value_write_buffer: NZUsize!(1024), + ordinal_write_buffer: NZUsize!(1024), + } + } + + #[test] + fn finalized_message_is_acknowledged_before_ready_peer_traffic() { + let executor = deterministic::Runner::default(); + executor.start(|mut context| async move { + let fixture = mocks::scheme_fixture_n(&mut context, 1); + let signer = ed25519::PrivateKey::from_seed(0); + let peer = ed25519::PrivateKey::from_seed(1).public_key(); + let participants = Set::from_iter_dedup([signer.public_key(), peer.clone()]); + let (_network, oracle) = Network::new_with_peers( + context.child("network"), + NetworkConfig { + max_size: 1024, + disconnect_on_block: true, + tracked_peer_sets: NZUsize!(1), + }, + vec![signer.public_key(), peer.clone()], + ) + .await; + let marshal = marshal_mailbox( + context.child("marshal"), + &signer, + fixture.schemes[0].clone(), + ) + .await; + let (fence, _gate) = Fence::new(Epoch::zero()); + let (mut actor, mut mailbox) = TestActor::new( + context.child("actor"), + Config { + signer: signer.clone(), + manager: oracle.manager(), + blocker: oracle.control(signer.public_key()), + participants_provider: StaticParticipants(participants), + secret_store: MemorySecretStore::default(), + strategy: Sequential, + registrar: mocks::MockConsumer::default(), + marshal, + state_sync_floor: None, + fence, + namespace: TEST_NAMESPACE, + sharing_mode: Mode::NonZeroCounter, + mailbox_size: NZUsize!(16), + partition_prefix: "dealing-priority-actor".into(), + max_participants: NZU32!(16), + blocks_per_epoch: NZU64!(2), + batch_verifier: PhantomData::, + }, + ); + + let mut store = Store::init( + context.child("store"), + "dealing-priority-store", + NZU32!(16), + MemorySecretStore::default(), + ) + .await; + let received = Arc::new(AtomicUsize::new(0)); + let receiver = QueuedReceiver { + peer: peer.clone(), + messages: (0..8).map(|_| IoBuf::from(vec![0xff])).collect(), + received: received.clone(), + }; + let (sender, _) = inert_channel([peer]); + let block = Arc::new(mocks::genesis_block(signer.public_key())); + let (ack, waiter) = Exact::handle(); + assert_eq!( + mailbox.report(marshal::Update::Block(block, ack)), + Feedback::Ok + ); + + let result = actor + .dealing(Epoch::zero(), &mut store, None, None, (sender, receiver)) + .await; + + assert!(result.is_continue()); + waiter + .await + .expect("finalized block should be acknowledged"); + assert_eq!(received.load(Ordering::SeqCst), 0); + }); + } +} diff --git a/glue/src/dkg/reshare/actor/dkg.rs b/glue/src/dkg/reshare/actor/dkg.rs new file mode 100644 index 00000000000..63697ae72ef --- /dev/null +++ b/glue/src/dkg/reshare/actor/dkg.rs @@ -0,0 +1,228 @@ +use super::setup::{EpochPreparation, PreparedEpoch}; +use crate::dkg::{ + ParticipantsProvider, Registrar, ReshareBlock, SecretStore, + reshare::{Actor, EpochInfoResponse, Message, actor::Mode, metrics::Phase, store::Store}, + types::Participants, +}; +use commonware_consensus::{ + marshal::core::Variant as MarshalVariant, + types::{Epoch, EpochPhase, Epocher, Height}, +}; +use commonware_cryptography::{ + BatchVerifier, Signer, bls12381::primitives::variant::Variant as BlsVariant, + certificate::Scheme, +}; +use commonware_macros::select_loop; +use commonware_p2p::{Blocker, Manager, Receiver, Sender, utils::mux::MuxHandle}; +use commonware_parallel::Strategy; +use commonware_runtime::{ + BufferPooler, Clock, Metrics, Spawner, Storage as RuntimeStorage, + telemetry::traces::TracedExt as _, +}; +use commonware_utils::{Acknowledgement, ordered::Set}; +use rand_core::CryptoRng; +use tracing::{debug, info_span}; + +impl Actor +where + E: Spawner + CryptoRng + Metrics + BufferPooler + Clock + RuntimeStorage, + B: ReshareBlock, + V: BlsVariant, + C: Signer, + M: Manager, + X: Blocker, + P: ParticipantsProvider, + SS: SecretStore, + T: Strategy, + BV: BatchVerifier + Send + 'static, + S: Scheme, + MV: MarshalVariant, + R: Registrar, + A: Acknowledgement, +{ + pub(super) async fn run_dkg( + &mut self, + store: &mut Store, + dealing_mux: &mut MuxHandle, + ) where + SE: Sender, + RE: Receiver, + { + let epoch = Epoch::zero(); + + // The one-shot DKG is never resumed or re-run. If this node already + // persisted its epoch-zero threshold share, the ceremony completed in a + // prior run and its artifacts were durably written then. Fail loudly + // rather than re-running the ceremony (which would misreport the completed + // DKG as a fresh failure once the chain has finalized past epoch zero). + if store.share(epoch).await.is_some() { + panic!( + "epoch-zero DKG already completed: this node's threshold share is \ + persisted, so the ceremony finished in a prior run and does not run \ + again. If the genesis artifact was not written, recover it from a peer \ + instead of re-running the DKG." + ); + } + + let completion = self.dkg_completion(); + let Some(mut prepared) = self.setup_dkg(store).await else { + self.complete_dkg(completion, store); + self.terminal().await; + return; + }; + + let chan = dealing_mux + .register(epoch.get()) + .await + .expect("failed to register DKG channel"); + + if prepared.phase == EpochPhase::Early + && self + .dealing( + epoch, + store, + prepared.dealer.as_mut(), + prepared.player.as_mut(), + chan, + ) + .await + .is_break() + { + return; + } + + if self + .inclusion(epoch, &prepared.info, store, prepared.dealer.as_mut()) + .await + .is_continue() + { + self.complete_dkg(completion, store); + self.terminal().await; + } + } + + async fn setup_dkg( + &mut self, + store: &mut Store, + ) -> Option> { + self.metrics.set_phase(Phase::Setup); + + let height = self + .marshal + .get_processed_height() + .await + .map_or_else(Height::zero, Height::next); + let bounds = self + .epocher + .containing(height) + .expect("epocher must know of block height"); + if bounds.epoch() != Epoch::zero() { + return None; + } + + let participants = self + .dkg_participants() + .expect("DKG setup requires DKG mode"); + let snapshot = Participants { + dealers: participants.clone(), + players: participants.clone(), + next_players: Set::default(), + }; + snapshot + .validate::(self.max_participants, None, 0) + .expect("DKG participants must be valid"); + snapshot + .validate_epoch_capacity::(self.blocks_per_epoch, None) + .expect("DKG epoch must have enough dealer-log slots"); + + let seed = store + .seed_or_random(Epoch::zero(), self.context.as_present_mut()) + .await; + store.put_seed(Epoch::zero(), seed).await; + + Some(self.prepare_epoch( + store, + EpochPreparation { + epoch: Epoch::zero(), + phase: bounds.phase(), + participants: snapshot, + previous: None, + share: None, + seed, + }, + )) + } + + async fn terminal(&mut self) { + select_loop! { + self.context, + on_stopped => { + debug!("shutdown signal received"); + return; + }, + Some(message) = self.mailbox.recv() else { + debug!("mailbox closed, shutting down"); + return; + } => match message { + Message::NextLog { span, response, .. } => { + let process = info_span!( + parent: &span, + "dkg.reshare.actor.dkg_terminal.next_log" + ); + process.in_scope(|| { + let _ = response.send(None); + }); + } + Message::ReleaseLog { .. } => {} + Message::EpochInfo { span, response, .. } => { + let process = info_span!( + parent: &span, + "dkg.reshare.actor.dkg_terminal.epoch_info" + ); + process.in_scope(|| { + let _ = response.send(EpochInfoResponse::Available(None)); + }); + } + Message::Finalized { + span, + response, + block, + } => { + let process = info_span!( + parent: &span, + "dkg.reshare.actor.dkg_terminal.finalized", + height = block.height().traced() + ); + process.in_scope(|| { + response.acknowledge(); + }); + } + }, + } + } + + pub(super) fn dkg_participants(&self) -> Option> { + match &self.mode { + Mode::Dkg { participants, .. } => Some(participants.clone()), + Mode::Reshare => None, + } + } + + fn dkg_completion(&mut self) -> Option> { + match &mut self.mode { + Mode::Dkg { completion, .. } => completion.take(), + Mode::Reshare => unreachable!("DKG completion requires DKG mode"), + } + } + + fn complete_dkg( + &mut self, + completion: Option>, + store: &mut Store, + ) { + let info = store.current().filter(|info| info.epoch == Epoch::zero()); + if let Some(completion) = completion { + completion(info); + } + } +} diff --git a/glue/src/dkg/reshare/actor/follower.rs b/glue/src/dkg/reshare/actor/follower.rs new file mode 100644 index 00000000000..5863738af00 --- /dev/null +++ b/glue/src/dkg/reshare/actor/follower.rs @@ -0,0 +1,124 @@ +use crate::dkg::{ + ParticipantsProvider, Registrar, ReshareBlock, SecretStore, + reshare::{Actor, EpochInfoResponse, Message, metrics::Phase, store::Store}, + types::Payload, +}; +use commonware_consensus::{marshal::core::Variant as MarshalVariant, types::Epocher}; +use commonware_cryptography::{ + BatchVerifier, Signer, bls12381::primitives::variant::Variant as BlsVariant, + certificate::Scheme, +}; +use commonware_macros::select_loop; +use commonware_p2p::{Blocker, Manager}; +use commonware_parallel::Strategy; +use commonware_runtime::{ + BufferPooler, Clock, Metrics, Spawner, Storage, telemetry::traces::TracedExt as _, +}; +use commonware_utils::{Acknowledgement, channel::fallible::OneshotExt}; +use rand_core::CryptoRng; +use std::ops::ControlFlow; +use tracing::{Instrument as _, debug, info_span}; + +impl Actor +where + E: Spawner + CryptoRng + Metrics + BufferPooler + Clock + Storage, + B: ReshareBlock, + V: BlsVariant, + C: Signer, + M: Manager, + X: Blocker, + P: ParticipantsProvider, + SS: SecretStore, + T: Strategy, + BV: BatchVerifier + Send + 'static, + S: Scheme, + MV: MarshalVariant, + R: Registrar, + A: Acknowledgement, +{ + /// Enter follower mode until the end of the current epoch is observed. + /// + /// This mode is entered when setup has no recoverable public protocol state. The actor cannot + /// participate in the active ceremony, so it waits until the final block. It registers the next + /// epoch as a signer only when a failed ceremony carries a locally held share forward; + /// otherwise, it registers as a verifier. + pub(super) async fn follow( + &mut self, + store: &mut Store, + ) -> ControlFlow<()> { + self.metrics.set_phase(Phase::Following); + + select_loop! { + self.context, + on_stopped => { + debug!("shutdown signal received"); + }, + Some(message) = self.mailbox.recv() else { + debug!("mailbox closed, shutting down"); + return ControlFlow::Break(()); + } => match message { + Message::NextLog { span, response, .. } => { + let process = info_span!(parent: &span, "dkg.reshare.actor.follower.next_log"); + process.in_scope(|| { + let _ = response.send_lossy(None); + }); + } + Message::ReleaseLog { .. } => {} + Message::EpochInfo { span, response, .. } => { + let process = + info_span!(parent: &span, "dkg.reshare.actor.follower.epoch_info"); + process.in_scope(|| { + let _ = response.send_lossy(EpochInfoResponse::Following); + }); + } + Message::Finalized { + span, + block, + response, + } => { + let process = info_span!( + parent: &span, + "dkg.reshare.actor.follower.finalized", + height = block.height().traced() + ); + let done = async { + let epoch_info = self + .epocher + .containing(block.height()) + .expect("epocher must know of epoch"); + if block.height() == epoch_info.last() { + let Some(Payload::EpochInfo(info)) = block.payload() else { + panic!( + "critical: boundary block {} does not contain EpochInfo for epoch {}", + block.height(), + epoch_info.epoch() + ); + }; + + let rng_seed = store + .seed_or_random(info.epoch, self.context.as_present_mut()) + .await; + let share = self.recovered_share(store, &info).await; + store + .commit_epoch(info.clone(), rng_seed, share.clone()) + .await; + self.register_epoch(&info, share).await; + + response.acknowledge(); + return true; + } + + response.acknowledge(); + false + } + .instrument(process) + .await; + if done { + return ControlFlow::Continue(()); + } + } + }, + } + ControlFlow::Break(()) + } +} diff --git a/glue/src/dkg/reshare/actor/inclusion.rs b/glue/src/dkg/reshare/actor/inclusion.rs new file mode 100644 index 00000000000..ca6a2f17ffc --- /dev/null +++ b/glue/src/dkg/reshare/actor/inclusion.rs @@ -0,0 +1,831 @@ +use crate::dkg::{ + ParticipantsProvider, Registrar, ReshareBlock, SecretStore, + reshare::{ + Actor, EpochInfoResponse, Message, + actor::Mode, + metrics::Phase, + store::{Dealer, Store}, + }, + types::{EpochInfo, EpochOutcome, Participants, Payload}, +}; +use commonware_consensus::{ + marshal::core::Variant as MarshalVariant, + types::{Epoch, EpochPhase, Epocher, FixedEpocher, Height}, +}; +use commonware_cryptography::{ + BatchVerifier, PublicKey, Signer, + bls12381::{ + dkg::feldman_desmedt::{DealerLog, Info, Logs, observe}, + primitives::{group::Share, variant::Variant as BlsVariant}, + }, + certificate::Scheme, +}; +use commonware_macros::{select, select_loop}; +use commonware_p2p::{Blocker, Manager}; +use commonware_parallel::Strategy; +use commonware_runtime::{ + BufferPooler, Clock, Metrics, Spawner, Storage as RuntimeStorage, signal, + telemetry::traces::TracedExt as _, +}; +use commonware_utils::{ + Acknowledgement, N3f1, + channel::{fallible::OneshotExt, oneshot}, + ordered::Set, +}; +use futures::StreamExt; +use rand_core::CryptoRng; +use std::{ + collections::BTreeMap, + num::{NonZeroU32, NonZeroU64}, + ops::ControlFlow, +}; +use tracing::{Instrument as _, debug, info, info_span, warn}; + +#[derive(Clone)] +struct Artifact { + info: EpochInfo, + share: Option, +} + +type PendingLogs = BTreeMap>; + +struct CachedArtifact { + // The cache is valid only for this exact effective view of finalized and + // pending logs. It lives for one inclusion phase and never owns durable + // protocol state. + logs: PendingLogs, + artifact: Option>, +} + +struct PendingLogScan<'a, V: BlsVariant, P> { + epoch: Epoch, + info: &'a Info, + epocher: FixedEpocher, + finalized_tip: Option, + final_height: Height, +} + +fn validate_future_participants( + participants: &Set

, + max_participants: NonZeroU32, + blocks_per_epoch: NonZeroU64, +) { + assert!( + !participants.is_empty(), + "participants provider returned empty future participant set" + ); + + let actual = participants.len(); + let max = max_participants.get() as usize; + assert!( + actual <= max, + "participants provider returned oversized future participant set: {actual} > {max}" + ); + + // Two epochs after this set is embedded it becomes both the dealer set + // and the previous output's player set, so its quorum bounds the dealer + // logs the ceremony must land on-chain. Reject an unusable provider set + // before it reaches a finalized EpochInfo, where the capacity violation + // would be re-derived from the chain and panic every node at the boundary. + Participants { + dealers: participants.clone(), + players: participants.clone(), + next_players: Set::default(), + } + .validate_epoch_capacity::(blocks_per_epoch, None) + .expect("participants provider returned set exceeding epoch dealer-log capacity"); +} + +/// The final block is special because proposal and verification may run ahead +/// of this actor's finalized-block reporter stream. In that case, the block +/// ancestry given to the application can contain pending dealer logs that are +/// not yet present in [`Store`]. +/// +/// Those pending logs must influence the final [`EpochInfo`] calculation so +/// proposal and verification agree with the block being evaluated. They must +/// not be persisted here: only the finalized reporter path below is durable. +/// This module therefore builds final artifacts from a temporary overlay of +/// finalized logs plus valid pending ancestry logs. +async fn pending_logs( + scan: PendingLogScan<'_, V, C::PublicKey>, + mut ancestry: crate::dkg::reshare::mailbox::ErasedAncestry, + mut shutdown: signal::Signal, + response: &mut oneshot::Sender>, +) -> Option> +where + B: ReshareBlock, + V: BlsVariant, + C: Signer, +{ + let mut blocks = Vec::new(); + loop { + let block = select! { + _ = &mut shutdown => return None, + _ = response.closed() => return None, + block = ancestry.next() => block, + }; + let Some(block) = block else { + break; + }; + let height = block.height(); + if scan.finalized_tip.is_some_and(|tip| height <= tip) { + break; + } + if height >= scan.final_height { + continue; + } + let Some(bounds) = scan.epocher.containing(height) else { + continue; + }; + if bounds.epoch() != scan.epoch { + continue; + } + if !matches!(bounds.phase(), EpochPhase::Midpoint | EpochPhase::Late) { + continue; + } + blocks.push(block); + } + + let mut logs = BTreeMap::new(); + for block in blocks.into_iter().rev() { + let height = block.height(); + let Some(Payload::DealerLog(log)) = block.payload() else { + continue; + }; + let Some((dealer, log)) = log.check(scan.info) else { + warn!(epoch = ?scan.epoch, ?height, "ignoring invalid pending dealer log"); + continue; + }; + logs.entry(dealer).or_insert(log); + } + Some(logs) +} + +impl Actor +where + E: Spawner + CryptoRng + Metrics + BufferPooler + Clock + RuntimeStorage, + B: ReshareBlock, + V: BlsVariant, + C: Signer, + M: Manager, + X: Blocker, + P: ParticipantsProvider, + SS: SecretStore, + T: Strategy, + BV: BatchVerifier + Send + 'static, + S: Scheme, + MV: MarshalVariant, + R: Registrar, + A: Acknowledgement, +{ + /// Run the inclusion phase for `epoch`. + /// + /// This phase begins at the epoch midpoint. It serves this node's finalized + /// dealer log to the application, re-offering it until it lands in a + /// finalized block, observes finalized dealer logs included by other + /// validators, and constructs the final epoch info when the application asks + /// to build or verify the epoch's final block. + /// + /// The phase returns after the finalized reporter delivers the epoch's last + /// block. At that point, any included final epoch info has been committed to + /// the store, the registrar has been updated, and the fence has been + /// unlocked for the next epoch. + pub(super) async fn inclusion( + &mut self, + epoch: Epoch, + info: &Info, + store: &mut Store, + mut dealer: Option<&mut Dealer>, + ) -> ControlFlow<()> { + self.metrics.set_phase(Phase::Inclusion); + + if let Some(dealer) = dealer.as_deref_mut() { + dealer.finalize::(); + } + + let mut served_at: Option = None; + let mut finalized_tip = self.marshal.get_processed_height().await; + let mut next_players = None; + let mut artifact_cache = None; + select_loop! { + self.context, + on_stopped => { + debug!("shutdown signal received"); + return ControlFlow::Break(()); + }, + Some(message) = self.mailbox.recv() else { + debug!("mailbox closed, shutting down"); + return ControlFlow::Break(()); + } => match message { + Message::NextLog { + span, + height, + release, + response, + } => { + let process = info_span!( + parent: &span, + "dkg.reshare.actor.inclusion.next_log", + height = height.traced() + ); + process.in_scope(|| { + let payload = served_at + .is_none() + .then(|| { + dealer + .as_ref() + .and_then(|dealer| dealer.finalized()) + .map(Payload::DealerLog) + }) + .flatten(); + let has_payload = payload.is_some(); + let reservation = payload + .map(|payload| crate::dkg::reshare::mailbox::LogReservation::new( + height, payload, release, + )); + if response.send_lossy(reservation) && has_payload { + served_at = Some(height); + } + }); + } + Message::ReleaseLog { height } => { + if served_at == Some(height) + && dealer + .as_ref() + .is_some_and(|dealer| dealer.finalized().is_some()) + { + served_at = None; + } + } + Message::EpochInfo { + span, + ancestry, + mut response, + } => { + if response.is_closed() { + continue; + } + let process = info_span!( + parent: &span, + "dkg.reshare.actor.inclusion.epoch_info" + ); + async { + let final_height = self + .epocher + .last(epoch) + .expect("epocher must know final epoch height"); + let scan = PendingLogScan { + epoch, + info, + epocher: self.epocher.clone(), + finalized_tip, + final_height, + }; + let Some(pending_logs) = pending_logs( + scan, + ancestry, + self.context.stopped(), + &mut response, + ) + .await + else { + return; + }; + if response.is_closed() { + return; + } + let artifact = self + .artifact( + epoch, + info, + store, + Some(&pending_logs), + &mut next_players, + &mut artifact_cache, + ) + .await; + let result = match artifact { + Some(artifact) => { + EpochInfoResponse::Available(Some(Payload::EpochInfo( + artifact.info, + ))) + } + None if matches!(self.mode, Mode::Dkg { .. }) => { + EpochInfoResponse::Available(None) + } + None => EpochInfoResponse::Unavailable, + }; + let _ = response.send_lossy(result); + } + .instrument(process) + .await; + } + Message::Finalized { + span, + block, + response, + } => { + let process = info_span!( + parent: &span, + "dkg.reshare.actor.inclusion.finalized", + height = block.height().traced() + ); + let done = async { + let bounds = self + .epocher + .containing(block.height()) + .expect("epocher must know of block height"); + assert_eq!( + bounds.epoch(), + epoch, + "inclusion received future epoch block" + ); + assert!( + matches!(bounds.phase(), EpochPhase::Midpoint | EpochPhase::Late), + "inclusion received block before midpoint" + ); + + let public_key = self.signer.public_key(); + Self::observe_dealer_log( + &public_key, + info, + store, + epoch, + dealer.as_deref_mut(), + block.payload(), + ) + .await; + + let done = block.height() == bounds.last(); + if done { + let artifact = self + .artifact( + epoch, + info, + store, + None, + &mut next_players, + &mut artifact_cache, + ) + .await; + self.handle_finalized_epoch_info( + epoch, + store, + artifact.as_ref(), + block.payload(), + ) + .await; + } + + finalized_tip = Some(block.height()); + + // Re-offer our dealer log if finalization reached the height + // we served it into without the log landing on-chain. When + // our log does finalize, observe_dealer_log above clears it + // via clear_finalized, so a still-present finalized log here + // means the proposal we served into lost the view. + if served_at.is_some_and(|served| block.height() >= served) + && dealer + .as_ref() + .is_some_and(|dealer| dealer.finalized().is_some()) + { + served_at = None; + } + + response.acknowledge(); + done + } + .instrument(process) + .await; + if done { + return ControlFlow::Continue(()); + } + } + }, + }; + + ControlFlow::Break(()) + } + + /// Persist a finalized dealer log from an included block. + /// + /// Invalid logs are ignored because the block has already passed + /// application verification. The finalized reporter path is the only place + /// where observed dealer logs become durable state. + pub(super) async fn observe_dealer_log( + public_key: &C::PublicKey, + info: &Info, + store: &mut Store, + epoch: Epoch, + dealer: Option<&mut Dealer>, + payload: Option>, + ) { + let Some(Payload::DealerLog(log)) = payload else { + return; + }; + let Some((dealer_key, log)) = log.check(info) else { + warn!(?epoch, "ignoring invalid dealer log"); + return; + }; + + // `log.check` only authenticates the self-signature, not dealer-set + // membership. A byzantine leader can embed a validly self-signed log from + // a key outside the round's dealer set in a finalized block. Such a log is + // never selected (selection filters non-dealers), so persisting it would + // only grow durable storage by one slot per attacker key. The round's + // dealers are the current output's players, so reject anything else. + if store + .current() + .is_some_and(|current| current.output.players().position(&dealer_key).is_none()) + { + warn!(?epoch, "ignoring dealer log from non-dealer"); + return; + } + + let ours = dealer_key == *public_key; + let stored = store.append_log(epoch, dealer_key.clone(), log).await; + info!( + ?epoch, + dealer = ?dealer_key, + ours, + stored, + "observed dealer log on chain" + ); + + if ours && let Some(dealer) = dealer { + dealer.clear_finalized(); + } + } + + /// Build the final epoch artifact from finalized state plus pending logs. + /// + /// The resulting [`EpochInfo`] is a lookahead for `epoch + 1`: its output is + /// the outcome of this epoch's reshare, its players are this epoch's + /// next players, and its next players are fetched for the following epoch. + /// + /// Artifact construction never mutates metrics or durable state. A + /// speculative result is cached only for the exact effective dealer-log map + /// and becomes authoritative only if a matching final block is finalized. + async fn artifact( + &mut self, + epoch: Epoch, + info: &Info, + store: &mut Store, + pending_logs: Option<&PendingLogs>, + next_players: &mut Option>, + artifact_cache: &mut Option>, + ) -> Option> { + let current = store.current(); + + // DKG mode is the only path that reaches inclusion without a current + // EpochInfo. In that case, the configured DKG participants are both the + // dealers and players for the one-shot ceremony. + let dkg_participants = if current.is_none() { + self.dkg_participants() + } else { + None + }; + if current.is_none() && dkg_participants.is_none() { + return None; + } + + let mut log_map = store.logs(epoch); + if let Some(pending_logs) = pending_logs { + for (dealer, log) in pending_logs { + log_map.entry(dealer.clone()).or_insert_with(|| log.clone()); + } + } + + if let Some(cached) = artifact_cache + .as_ref() + .filter(|cached| cached.logs == log_map) + { + return cached.artifact.clone(); + } + + let mut logs = Logs::<_, _, N3f1>::new(info.clone()); + for (dealer, log) in log_map.clone() { + logs.record(dealer, log); + } + + let public_key = self.signer.public_key(); + let players = current + .as_ref() + .map(|current| current.players.clone()) + .or(dkg_participants.clone()) + .expect("current epoch or DKG mode must provide players"); + let player = players.position(&public_key).and_then(|_| { + store.create_player_with_logs::( + epoch, + self.signer.clone(), + info.clone(), + &log_map, + ) + }); + + let outcome = if let Some(player) = player { + match player.finalize::(self.context.as_present_mut(), logs, &self.strategy) { + Ok((output, share)) => Some((output, Some(share))), + Err(error) => { + warn!(?epoch, ?error, "failed to finalize player"); + None + } + } + } else { + match observe::<_, _, N3f1, BV>(self.context.as_present_mut(), logs, &self.strategy) { + Ok(output) => Some((output, None)), + Err(error) => { + warn!(?epoch, ?error, "failed to observe reshare outcome"); + None + } + } + }; + + let future_players = if current.is_some() { + match next_players { + Some(players) => players.clone(), + None => { + // The provider contract requires this set to remain stable + // for the epoch, so reuse one lookup across competing final + // block proposals and verification attempts. + let players = self + .participants_provider + .participants(epoch.next().next()) + .await; + validate_future_participants::( + &players, + self.max_participants, + self.blocks_per_epoch, + ); + *next_players = Some(players.clone()); + players + } + } + } else { + Default::default() + }; + + let artifact = match outcome { + Some((output, share)) => match current { + Some(current) => { + let next_epoch = epoch.next(); + Some(Artifact { + info: EpochInfo { + outcome: EpochOutcome::Success, + epoch: next_epoch, + output, + players: current.next_players, + next_players: future_players, + }, + share, + }) + } + None => { + // DKG success emits the genesis threshold artifact directly. + // There is no next committee to prefetch because this + // one-shot chain terminates after epoch zero. + let share = share.expect("DKG participant must receive a share"); + Some(Artifact { + info: EpochInfo { + outcome: EpochOutcome::Success, + epoch, + output, + players, + next_players: future_players, + }, + share: Some(share), + }) + } + }, + None => { + let Some(current) = current else { + *artifact_cache = Some(CachedArtifact { + logs: log_map, + artifact: None, + }); + return None; + }; + let share = if current.output.players().position(&public_key).is_some() { + store.share(epoch).await + } else { + None + }; + Some(Artifact { + info: EpochInfo { + outcome: EpochOutcome::Failure, + epoch: epoch.next(), + output: current.output, + players: current.next_players, + next_players: future_players, + }, + share, + }) + } + }; + + *artifact_cache = Some(CachedArtifact { + logs: log_map, + artifact: artifact.clone(), + }); + artifact + } + + /// Commit finalized epoch info and configure the next epoch. + /// + /// The final block must carry epoch info for the next epoch. If the locally + /// reconstructed artifact matches it, this node also persists its new share. + /// If not, the epoch info is still committed without a share so the node can + /// enter the next epoch as a verifier. + async fn handle_finalized_epoch_info( + &mut self, + epoch: Epoch, + store: &mut Store, + artifact: Option<&Artifact>, + payload: Option>, + ) { + let dkg = matches!(self.mode, Mode::Dkg { .. }); + if dkg && payload.is_none() { + // A failed one-shot DKG has no artifact to commit, so the final + // block intentionally carries no EpochInfo. Continuous reshare + // never permits this because its final block must always carry the + // next epoch pointer. + assert!( + artifact.is_none(), + "final block omitted DKG info despite locally reconstructing it" + ); + return; + } + + let Some(Payload::EpochInfo(info)) = payload else { + panic!("final block missing epoch info for epoch {epoch:?}"); + }; + let next_epoch = if dkg { epoch } else { epoch.next() }; + assert_eq!( + info.epoch, next_epoch, + "final block carried epoch info for wrong epoch" + ); + + // Record only canonical finalized outcomes. Speculative artifact + // construction is intentionally side-effect free. + match info.outcome { + EpochOutcome::Success => self + .metrics + .record_success(&info.output, &self.signer.public_key()), + EpochOutcome::Failure => { + self.metrics.failed_epochs.inc(); + } + } + + let share = artifact + .filter(|artifact| artifact.info == info) + .and_then(|artifact| artifact.share.clone()); + let rng_seed = store + .seed_or_random(next_epoch, self.context.as_present_mut()) + .await; + store + .commit_epoch(info.clone(), rng_seed, share.clone()) + .await; + info!( + epoch = ?info.epoch, + round = info.epoch.get(), + success = matches!(info.outcome, EpochOutcome::Success), + dealers = ?info.output.dealers(), + players = ?info.players, + next_players = ?info.next_players, + "completed reshare ceremony" + ); + if !dkg { + self.register_epoch(&info, share).await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dkg::tests::mocks::{TestBlock, TestBlsVariant}; + use commonware_cryptography::{ + Signer, + bls12381::primitives::sharing::Mode as SharingMode, + ed25519::{PrivateKey, PublicKey}, + }; + use commonware_runtime::{Runner, Spawner, Supervisor, deterministic}; + use commonware_utils::{N3f1, NZU32, NZU64, channel::oneshot, ordered::Set}; + use futures::{FutureExt, stream}; + use std::sync::Arc; + + type TestResponse = EpochInfoResponse; + + fn signers() -> Vec { + (0..4).map(PrivateKey::from_seed).collect() + } + + fn players() -> Set { + Set::from_iter_dedup(signers().iter().map(Signer::public_key)) + } + + fn info() -> Info { + Info::new::( + b"_COMMONWARE_GLUE_DKG_RESHARE_INCLUSION_TEST", + 0, + None, + SharingMode::NonZeroCounter, + players(), + players(), + ) + .expect("valid info") + } + + fn stalled_ancestry() -> crate::dkg::reshare::mailbox::ErasedAncestry { + Box::pin(stream::pending::>()) + } + + fn scan( + info: &Info, + ) -> PendingLogScan<'_, TestBlsVariant, PublicKey> { + PendingLogScan { + epoch: Epoch::zero(), + info, + epocher: FixedEpocher::new(NZU64!(4)), + finalized_tip: None, + final_height: Height::new(3), + } + } + + #[test] + #[should_panic(expected = "participants provider returned empty future participant set")] + fn future_participants_rejects_empty_provider_set() { + validate_future_participants::( + &Set::::default(), + NZU32!(4), + NZU64!(8), + ); + } + + #[test] + #[should_panic(expected = "participants provider returned oversized future participant set")] + fn future_participants_rejects_oversized_provider_set() { + validate_future_participants::(&players(), NZU32!(3), NZU64!(8)); + } + + #[test] + fn future_participants_accepts_set_within_epoch_capacity() { + // Four participants need a three-log dealer quorum; an eight-block + // epoch has three inclusion slots. + validate_future_participants::(&players(), NZU32!(4), NZU64!(8)); + } + + #[test] + #[should_panic(expected = "exceeding epoch dealer-log capacity")] + fn future_participants_rejects_set_exceeding_epoch_capacity() { + // Four participants need a three-log dealer quorum, but a four-block + // epoch only has one inclusion slot. + validate_future_participants::(&players(), NZU32!(4), NZU64!(4)); + } + + #[test] + fn pending_logs_cancels_stalled_ancestry_when_response_closes() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let info = info(); + let (mut response_tx, response_rx) = oneshot::channel::(); + let pending = pending_logs( + scan(&info), + stalled_ancestry(), + context.stopped(), + &mut response_tx, + ); + futures::pin_mut!(pending); + assert!(pending.as_mut().now_or_never().is_none()); + + drop(response_rx); + + assert!(pending.await.is_none()); + }); + } + + #[test] + fn pending_logs_cancels_stalled_ancestry_when_runtime_stops() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let info = info(); + let (mut response_tx, _response_rx) = oneshot::channel::(); + let pending = pending_logs( + scan(&info), + stalled_ancestry(), + context.stopped(), + &mut response_tx, + ); + futures::pin_mut!(pending); + assert!(pending.as_mut().now_or_never().is_none()); + + let stopper = context.child("stopper"); + let stop = context.child("stop").spawn(|_| async move { + stopper.stop(0, None).await.expect("runtime should stop"); + }); + + assert!(pending.await.is_none()); + stop.await.expect("stop task should finish"); + }); + } +} diff --git a/glue/src/dkg/reshare/actor/mod.rs b/glue/src/dkg/reshare/actor/mod.rs new file mode 100644 index 00000000000..8dcb5ff5950 --- /dev/null +++ b/glue/src/dkg/reshare/actor/mod.rs @@ -0,0 +1,451 @@ +//! Drive per-epoch BLS resharing from finalized marshal state. +//! +//! The actor bridges finalized epoch metadata, the Feldman-Desmedt reshare +//! protocol, P2P dealer traffic, and certificate-scheme registration. Each loop +//! iteration derives the active epoch from marshal's processed height, loads the +//! epoch's public [`EpochInfo`] from the finalized +//! boundary block, and either participates in the ceremony or follows until the +//! next boundary is finalized. +//! +//! # Epoch Lifecycle +//! +//! A participating epoch has three states: +//! +//! 1. **Setup** reads the canonical boundary block, replays durable recovery +//! state, opens the epoch peer set, registers the current scheme with the +//! [`Registrar`], and prepares optional dealer/player state for this node. +//! 2. **Dealing** runs during the early half of the epoch. Dealers send private +//! shares to players over P2P and players return signed acknowledgements. +//! 3. **Inclusion** runs from the midpoint through the final block. The actor +//! offers one finalized dealer log to the application, observes finalized +//! logs on-chain, computes the next [`EpochInfo`], +//! and registers the next epoch once that boundary block finalizes. +//! +//! ```text +//! finalized boundary for epoch N +//! | +//! v +//! setup: load EpochInfo(N), share, seed, recovery journal +//! | +//! +-- no boundary info and already inside epoch --> follower mode +//! | +//! v +//! early blocks +//! | +//! v +//! dealing: dealer shares <--> player acknowledgements +//! | +//! v +//! midpoint +//! | +//! v +//! inclusion: propose/observe dealer logs +//! | +//! v +//! final block carries EpochInfo(N + 1) +//! | +//! v +//! register scheme for epoch N + 1 +//! ``` +//! +//! # Payload Flow +//! +//! Consensus asks the actor for an optional payload before proposing each block, +//! and reports finalized blocks after marshal processes them: +//! +//! ```text +//! application --Next(height)-----------> Actor --Payload?----------> application +//! marshal --Finalized(block)-------> Actor --acknowledge-------> marshal +//! peer --Dealer/Ack(epoch)------> Actor --Ack/Dealer(epoch)-> peer +//! ``` +//! +//! During dealing, `Next` never returns a payload. During inclusion, `Next` +//! returns at most one dealer log before the final height, and returns the +//! computed [`EpochInfo`] at the final height when +//! enough valid logs are available. Finalized blocks are the source of truth: +//! only logs and epoch info that appear in finalized blocks update durable state +//! or registered schemes. +//! +//! # Crash Recovery +//! +//! Recovery state is split by sensitivity. Public, replayable protocol messages +//! are journaled by [`Store`]: dealer public messages, player acknowledgements, +//! and finalized dealer logs. Secret material is kept only in [`SecretStore`]: +//! current shares, private dealings, and dealer RNG seeds. Public epoch info is +//! not cached in the recovery journal because it is re-derived from finalized +//! boundary blocks on startup. +//! +//! ```text +//! restart +//! | +//! +--> marshal processed height determines candidate epoch +//! | +//! +--> boundary block supplies canonical EpochInfo +//! | +//! +--> Store replays public journal +//! | +//! +--> SecretStore supplies share, private dealings, and seed +//! | +//! v +//! resume as dealer/player/observer when enough state is available +//! ``` +//! +//! Reusing the persisted dealer seed makes regenerated dealer shares identical +//! after a restart. Persisted acknowledgements and finalized logs let a player or +//! observer rebuild the same outcome even though P2P messages and finalized-block +//! notifications are not replayed by the runtime. If the node lacks a valid share +//! for a dealer role, it simply observes or plays instead of manufacturing local +//! state. +//! +//! # Follower Mode +//! +//! The actor follows instead of participating when setup cannot read the boundary +//! [`EpochInfo`] for the epoch containing marshal's next unprocessed height. This can occur during +//! state-sync handoff because marshal may not retain the preceding boundary block. +//! +//! ```text +//! processed height + 1 = H +//! | +//! v +//! H is in epoch N +//! | +//! v +//! boundary EpochInfo(N) unavailable locally +//! | +//! v +//! follower mode until final(N) +//! | +//! v +//! final(N) carries EpochInfo(N + 1) +//! | +//! +--> failed ceremony and prior share held -> register signer +//! | +//! +--> otherwise -> register verifier +//! | +//! v +//! setup again +//! ``` +//! +//! While following, `Next` always returns no payload and finalized blocks are +//! acknowledged without mutation until the final block of the current epoch. The +//! final block's epoch info is used as the next loop's boundary state. When a +//! failed ceremony carries the previous threshold output forward, the actor also +//! carries forward a locally held share and registers as a signer. Otherwise, it +//! commits without a share and registers as a verifier. + +use crate::dkg::{ + ParticipantsProvider, Registrar, ReshareBlock, SecretStore, + fence::Fence, + reshare::{Mailbox, Message, metrics::Metrics as ReshareMetrics, store::Store}, + types::EpochInfo, +}; +use commonware_actor::mailbox::{self as actor_mailbox, Receiver as MailboxReceiver}; +use commonware_consensus::{ + marshal::core::{CommitmentFallback, Mailbox as MarshalMailbox, Variant as MarshalVariant}, + types::{EpochPhase, FixedEpocher}, +}; +use commonware_cryptography::{ + BatchVerifier, PublicKey, Signer, + bls12381::primitives::{sharing::Mode as SharingMode, variant::Variant as BlsVariant}, + certificate::Scheme, +}; +use commonware_p2p::{Blocker, Manager, Receiver, Sender, utils::mux::Muxer}; +use commonware_parallel::Strategy; +use commonware_runtime::{ + BufferPooler, Clock, ContextCell, Handle, Metrics, Spawner, Storage, spawn_cell, +}; +use commonware_utils::{Acknowledgement, acknowledgement::Exact, ordered::Set}; +use rand_core::CryptoRng; +use std::{ + marker::PhantomData, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, +}; + +type DkgCompletion = Box>) + Send>; + +mod dealing; +mod dkg; +mod follower; +mod inclusion; +mod setup; +use setup::Setup; + +/// Configuration for the crate-private one-shot DKG mode. +pub(crate) struct DkgConfig +where + V: BlsVariant, + P: PublicKey, +{ + pub(crate) participants: Set

, + pub(crate) completion: DkgCompletion, +} + +enum Mode +where + V: BlsVariant, + P: PublicKey, +{ + Reshare, + Dkg { + participants: Set

, + completion: Option>, + }, +} + +/// Configuration for [`Actor`]. +pub struct Config +where + C: Signer, + X: Blocker, + S: Scheme, + MV: MarshalVariant, +{ + /// Signer for player acknowledgments and dealer logs. + pub signer: C, + + /// P2P manager used to track peers during one-shot DKG. + /// + /// Continuous reshare peer tracking is owned by + /// [`orchestrator`](crate::dkg::orchestrator). + pub manager: M, + + /// Blocker used to block peers that send invalid protocol messages. + pub blocker: X, + + /// Provider of participant policy. + pub participants_provider: P, + + /// Store for private share material. + pub secret_store: SS, + + /// Parallel strategy for cryptographic verification. + pub strategy: T, + + /// Registrar for configuring signing scheme providers. + pub registrar: R, + + /// Marshal mailbox used to read canonical public epoch state from finalized + /// boundary blocks. + pub marshal: MarshalMailbox, + + /// State sync floor commitment, if marshal is starting from a floor. + pub state_sync_floor: Option, + + /// Epoch readiness fence. + pub fence: Fence, + + /// Application namespace for transcript separation. + pub namespace: &'static [u8], + + /// Sharing mode used for newly generated threshold outputs. + pub sharing_mode: SharingMode, + + /// Actor mailbox capacity. + pub mailbox_size: NonZeroUsize, + + /// Runtime-storage partition prefix. + pub partition_prefix: String, + + /// Maximum participants accepted in decoded protocol values and + /// provider-supplied future participant sets. + pub max_participants: NonZeroU32, + + /// Epoch schedule used to interpret finalized block heights. + pub blocks_per_epoch: NonZeroU64, + + /// Batch verifier marker. + pub batch_verifier: PhantomData, +} + +pub struct Actor +where + E: Spawner + CryptoRng + Metrics + BufferPooler + Clock + Storage, + B: ReshareBlock, + V: BlsVariant, + C: Signer, + M: Manager, + X: Blocker, + P: ParticipantsProvider, + SS: SecretStore, + T: Strategy, + BV: BatchVerifier + Send + 'static, + S: Scheme, + MV: MarshalVariant, + R: Registrar, + A: Acknowledgement, +{ + context: ContextCell, + mailbox: MailboxReceiver>, + signer: C, + manager: M, + blocker: X, + participants_provider: P, + secret_store: Option, + strategy: T, + registrar: R, + marshal: MarshalMailbox, + state_sync_floor: Option, + fence: Fence, + namespace: &'static [u8], + sharing_mode: SharingMode, + partition_prefix: String, + max_participants: NonZeroU32, + blocks_per_epoch: NonZeroU64, + epocher: FixedEpocher, + metrics: ReshareMetrics, + mode: Mode, + batch_verifier: PhantomData, +} + +impl Actor +where + E: Spawner + CryptoRng + Metrics + BufferPooler + Clock + Storage, + B: ReshareBlock, + V: BlsVariant, + C: Signer, + M: Manager, + X: Blocker, + P: ParticipantsProvider, + SS: SecretStore, + T: Strategy, + BV: BatchVerifier + Send + 'static, + S: Scheme, + MV: MarshalVariant, + R: Registrar, + A: Acknowledgement, +{ + pub fn new( + context: E, + config: Config, + ) -> (Self, Mailbox) { + let epocher = FixedEpocher::new(config.blocks_per_epoch); + let (sender, mailbox) = actor_mailbox::new(context.child("mailbox"), config.mailbox_size); + let metrics = ReshareMetrics::new(&context); + ( + Self { + context: ContextCell::new(context), + mailbox, + signer: config.signer, + manager: config.manager, + blocker: config.blocker, + participants_provider: config.participants_provider, + secret_store: Some(config.secret_store), + strategy: config.strategy, + registrar: config.registrar, + marshal: config.marshal, + state_sync_floor: config.state_sync_floor, + fence: config.fence, + namespace: config.namespace, + sharing_mode: config.sharing_mode, + partition_prefix: config.partition_prefix, + max_participants: config.max_participants, + blocks_per_epoch: config.blocks_per_epoch, + epocher, + metrics, + mode: Mode::Reshare, + batch_verifier: config.batch_verifier, + }, + Mailbox::new(sender), + ) + } + + pub(crate) fn new_dkg( + context: E, + config: Config, + dkg: DkgConfig, + ) -> (Self, Mailbox) { + let (mut actor, mailbox) = Self::new(context, config); + actor.mode = Mode::Dkg { + participants: dkg.participants, + completion: Some(dkg.completion), + }; + (actor, mailbox) + } + + pub fn start(mut self, chan: (SE, RE)) -> Handle<()> + where + SE: Sender, + RE: Receiver, + { + spawn_cell!(self.context, self.run(chan)) + } + + async fn run(mut self, (sender, receiver): (SE, RE)) + where + SE: Sender, + RE: Receiver, + { + let secret_store = self + .secret_store + .take() + .expect("secret store must be available when actor starts"); + let mut store = Store::init( + self.context.child("store"), + &self.partition_prefix, + self.max_participants, + secret_store, + ) + .await; + + let (mux, mut dealing_mux) = Muxer::new(self.context.child("mux"), sender, receiver, 128); + mux.start(); + + if let Some(commitment) = self.state_sync_floor.take() { + self.marshal + .subscribe_by_commitment(commitment, CommitmentFallback::Wait) + .await + .expect("marshal must yield state sync floor block"); + } + + if matches!(self.mode, Mode::Dkg { .. }) { + self.run_dkg(&mut store, &mut dealing_mux).await; + return; + } + + let mut current_epoch = None; + loop { + let Some(prepared) = self.setup(&mut store, current_epoch.take()).await else { + return; + }; + let Setup::Participate(prepared) = prepared else { + if self.follow(&mut store).await.is_break() { + return; + } + current_epoch = store.current().map(|info| info.epoch); + continue; + }; + let mut prepared = *prepared; + + let chan = dealing_mux + .register(prepared.epoch.get()) + .await + .expect("failed to register reshare epoch channel"); + + if prepared.phase == EpochPhase::Early { + let dealer = prepared.dealer.as_mut(); + let player = prepared.player.as_mut(); + if self + .dealing(prepared.epoch, &mut store, dealer, player, chan) + .await + .is_break() + { + return; + } + } + + if self + .inclusion( + prepared.epoch, + &prepared.info, + &mut store, + prepared.dealer.as_mut(), + ) + .await + .is_break() + { + return; + } + current_epoch = Some(prepared.epoch.next()); + } + } +} diff --git a/glue/src/dkg/reshare/actor/setup.rs b/glue/src/dkg/reshare/actor/setup.rs new file mode 100644 index 00000000000..08061f0ebee --- /dev/null +++ b/glue/src/dkg/reshare/actor/setup.rs @@ -0,0 +1,261 @@ +use crate::dkg::{ + ParticipantsProvider, Registrar, ReshareBlock, SecretStore, + reshare::{ + Actor, + actor::Mode, + metrics::Phase, + store::{Dealer, Player, Store}, + }, + types::{EpochInfo, EpochOutcome, Participants, Payload, SchemeInfo}, +}; +use commonware_consensus::{ + marshal::{Identifier, core::Variant as MarshalVariant}, + types::{Epoch, EpochPhase, Epocher, Height}, +}; +use commonware_cryptography::{ + BatchVerifier, PublicKey, Signer, + bls12381::{ + dkg::feldman_desmedt::{Info, Output}, + primitives::{group::Share, variant::Variant as BlsVariant}, + }, + certificate::Scheme, + transcript::Summary, +}; +use commonware_p2p::{Blocker, Manager}; +use commonware_parallel::Strategy; +use commonware_runtime::{ + BufferPooler, Clock, Metrics, Spawner, Storage, telemetry::metrics::GaugeExt, +}; +use commonware_utils::{Acknowledgement, N3f1}; +use rand_core::CryptoRng; + +pub(super) struct PreparedEpoch +where + V: BlsVariant, + C: Signer, +{ + pub(super) epoch: Epoch, + pub(super) phase: EpochPhase, + pub(super) info: Info, + pub(super) dealer: Option>, + pub(super) player: Option>, +} + +pub(super) struct EpochPreparation +where + V: BlsVariant, + P: PublicKey, +{ + pub(super) epoch: Epoch, + pub(super) phase: EpochPhase, + pub(super) participants: Participants

, + pub(super) previous: Option>, + pub(super) share: Option, + pub(super) seed: Summary, +} + +pub(super) enum Setup +where + V: BlsVariant, + C: Signer, +{ + Follow, + Participate(Box>), +} + +impl Actor +where + E: Spawner + CryptoRng + Metrics + BufferPooler + Clock + Storage, + B: ReshareBlock, + V: BlsVariant, + C: Signer, + M: Manager, + X: Blocker, + P: ParticipantsProvider, + SS: SecretStore, + T: Strategy, + BV: BatchVerifier + Send + 'static, + S: Scheme, + MV: MarshalVariant, + R: Registrar, + A: Acknowledgement, +{ + pub(super) async fn setup( + &mut self, + store: &mut Store, + current_epoch: Option, + ) -> Option> { + self.metrics.set_phase(Phase::Setup); + + let height = match current_epoch { + Some(epoch) => self + .epocher + .first(epoch) + .expect("epocher must know hinted epoch"), + None => self + .marshal + .get_processed_height() + .await + .map_or_else(Height::zero, Height::next), + }; + let bounds = self + .epocher + .containing(height) + .expect("epocher must know of block height"); + let epoch = bounds.epoch(); + + let current = store.current().filter(|current| current.epoch == epoch); + let already_committed = current.is_some(); + let info = match current { + Some(info) => info, + None => { + let Some(info) = self.boundary_epoch_info(epoch).await else { + return Some(Setup::Follow); + }; + info + } + }; + if info.epoch != epoch { + panic!( + "boundary epoch info describes epoch {}, expected {epoch}", + info.epoch + ); + } + + let participants = info.participants(); + let round = epoch.get(); + participants + .validate(self.max_participants, Some(&info.output), round) + .expect("boundary epoch participants must be valid"); + participants + .validate_epoch_capacity(self.blocks_per_epoch, Some(&info.output)) + .expect("boundary epoch must have enough dealer-log slots"); + + let share = self.recovered_share(store, &info).await; + let seed = store + .seed_or_random(epoch, self.context.as_present_mut()) + .await; + if !already_committed { + store.commit_epoch(info.clone(), seed, share.clone()).await; + self.register_epoch(&info, share.clone()).await; + } + store.prune(epoch.previous().unwrap_or(epoch)).await; + + Some(Setup::Participate(Box::new(self.prepare_epoch( + store, + EpochPreparation { + epoch, + phase: bounds.phase(), + participants, + previous: Some(info.output.clone()), + share, + seed, + }, + )))) + } + + pub(super) async fn recovered_share( + &mut self, + store: &mut Store, + info: &EpochInfo, + ) -> Option { + let share = store.share(info.epoch).await; + if share.is_some() || info.outcome != EpochOutcome::Failure { + return share; + } + + info.output.players().position(&self.signer.public_key())?; + + let previous = info.epoch.previous()?; + store.share(previous).await + } + + async fn boundary_epoch_info(&mut self, epoch: Epoch) -> Option> { + let height = epoch + .previous() + .and_then(|e| self.epocher.last(e)) + .unwrap_or(Height::zero()); + let block = self + .marshal + .get_block(Identifier::Height(height)) + .await + .map(MV::into_inner)?; + let Some(Payload::EpochInfo(info)) = block.payload() else { + panic!("boundary block {height} missing epoch info"); + }; + Some(info) + } + + pub(super) async fn register_epoch( + &mut self, + info: &EpochInfo, + share: Option, + ) { + let scheme_info = share.map_or_else( + || SchemeInfo::Verifier { + participants: info.output.players().clone(), + sharing: info.output.public().clone(), + }, + |share| SchemeInfo::Signer { + participants: info.output.players().clone(), + sharing: info.output.public().clone(), + share, + }, + ); + self.registrar.register(info.epoch, scheme_info).await; + self.fence.mark(info.epoch); + } + + pub(super) fn prepare_epoch( + &mut self, + store: &mut Store, + preparation: EpochPreparation, + ) -> PreparedEpoch { + let EpochPreparation { + epoch, + phase, + participants, + previous, + share, + seed, + } = preparation; + + let round = epoch.get(); + if matches!(&self.mode, Mode::Dkg { .. }) { + let _ = self + .manager + .track(epoch.get(), participants.tracked_peers()); + } + let _ = self.metrics.current_epoch.try_set(epoch.get() as i64); + let _ = self.metrics.current_round.try_set(round as i64); + + let round = Info::new::( + self.namespace, + round, + previous.clone(), + self.sharing_mode, + participants.dealers.clone(), + participants.players.clone(), + ) + .expect("epoch participants must produce valid round info"); + + let public_key = self.signer.public_key(); + let has_prior_share = previous.is_none() || share.is_some(); + let dealer = if participants.dealers.position(&public_key).is_some() && has_prior_share { + store.create_dealer::(epoch, self.signer.clone(), round.clone(), share, seed) + } else { + None + }; + let player = participants.players.position(&public_key).and_then(|_| { + store.create_player::(epoch, self.signer.clone(), round.clone()) + }); + + PreparedEpoch { + epoch, + phase, + info: round, + dealer, + player, + } + } +} diff --git a/glue/src/dkg/reshare/application.rs b/glue/src/dkg/reshare/application.rs new file mode 100644 index 00000000000..18365d88e3a --- /dev/null +++ b/glue/src/dkg/reshare/application.rs @@ -0,0 +1,869 @@ +use crate::dkg::{ + ReshareBlock, + reshare::{EpochInfoResponse, Mailbox}, + types::Payload, +}; +use commonware_consensus::{ + Application as ConsensusApplication, CertifiableBlock, + marshal::ancestry::Ancestry, + types::{EpochPhase, Epocher as _, FixedEpocher, Height}, +}; +use commonware_cryptography::{Signer, bls12381::primitives::variant::Variant}; +use commonware_runtime::{Clock, Metrics, Spawner, telemetry::traces::TracedExt as _}; +use rand_core::Rng; +use std::{future, num::NonZeroU64}; +use tracing::{debug, field}; + +/// Per-proposal input handed to an application wrapped by [`Application`]. +/// +/// Carries the wrapper's parent input alongside the reshare `payload` selected +/// and fetched for the block being proposed. The wrapped application attaches +/// `payload` to the block it builds and uses `parent` for its own purposes. +pub struct Input { + /// Input forwarded from the application wrapping the reshare wrapper. + pub parent: Parent, + + /// The reshare payload selected for this proposal, if any. + pub payload: Option>, +} + +/// An [`Application`](commonware_consensus::Application) wrapper that enforces the +/// reshare block-validity contract and drives the reshare payload for proposals. +/// +/// When the reshare actor tracks an epoch's ceremony, the wrapper rejects a +/// final block whose payload differs from the independently reconstructed +/// [`EpochInfo`](crate::dkg::types::EpochInfo). An actor that starts following +/// mid-epoch lacks the protocol history required for that comparison, so the +/// wrapper delegates verification to the inner application rather than treating +/// missing local state as an invalid proposal. The wrapper always rejects stray +/// payloads carried by non-final blocks in the early dealing window. +/// +/// For proposals, the wrapper selects and fetches the payload for the block being +/// built (a dealer log from the midpoint onward, the epoch info on the final +/// block) and hands it to the inner application through [`Input`], so the +/// inner application neither talks to the reshare mailbox nor tracks epoch +/// boundaries. It only attaches the handed-over payload to the block it builds, +/// because the wrapper cannot build the application's block type itself. +/// +/// The wrapper is a plain [`Application`](commonware_consensus::Application), so +/// it composes with any consensus application, including one adapted through +/// [`stateful`](crate::stateful). It forwards its own parent input to the inner +/// application as [`Input::parent`], so nesting under another +/// input-providing application still works. +pub struct Application +where + B: ReshareBlock, + V: Variant, + C: Signer, +{ + inner: A, + reshare: Mailbox, + epocher: FixedEpocher, +} + +impl Application +where + B: ReshareBlock, + V: Variant, + C: Signer, +{ + /// Wraps `inner`, using `reshare` to select final-block epoch info and dealer + /// logs and `blocks_per_epoch` to locate epoch boundaries and phases. + pub const fn new(inner: A, reshare: Mailbox, blocks_per_epoch: NonZeroU64) -> Self { + Self { + inner, + reshare, + epocher: FixedEpocher::new(blocks_per_epoch), + } + } + + fn final_block(&self, height: Height) -> bool { + self.epocher + .containing(height) + .is_some_and(|info| info.last() == height) + } + + fn phase(&self, height: Height) -> Option { + self.epocher.containing(height).map(|info| info.phase()) + } +} + +impl Clone for Application +where + A: Clone, + B: ReshareBlock, + V: Variant, + C: Signer, +{ + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + reshare: self.reshare.clone(), + epocher: self.epocher.clone(), + } + } +} + +impl ConsensusApplication for Application +where + E: Rng + Spawner + Metrics + Clock, + A: ConsensusApplication>, + A::Context: Send, + B: ReshareBlock + CertifiableBlock + Clone, + V: Variant, + C: Signer, + I: Send, +{ + type SigningScheme = A::SigningScheme; + type Context = A::Context; + type Block = A::Block; + type Input = I; + + #[tracing::instrument( + name = "dkg.reshare.application.propose", + level = "info", + skip_all, + fields( + height = field::Empty, + phase = field::Empty, + has_payload = field::Empty + ) + )] + async fn propose( + &mut self, + context: (E, Self::Context), + ancestry: impl Ancestry, + input: Self::Input, + ) -> Option { + // Select and fetch the payload for the block being built, then hand it to + // the inner application alongside its own input. + let Some(parent) = ancestry.peek() else { + debug!("proposal rejected: missing parent ancestry"); + return None; + }; + let height = parent.height().next(); + let phase = self.phase(height); + let span = tracing::Span::current(); + span.record("height", height.traced()); + span.record("phase", field::debug(phase)); + + let (payload, log_reservation) = if self.final_block(height) { + match self.reshare.epoch_info(ancestry.clone()).await { + EpochInfoResponse::Available(payload) => (payload, None), + EpochInfoResponse::Pending => { + debug!("proposal skipped: final block epoch info is not ready"); + return None; + } + EpochInfoResponse::Following => { + debug!("proposal skipped: follower has no final block epoch info"); + return None; + } + EpochInfoResponse::Unavailable => { + debug!("proposal skipped: final block epoch info is unavailable"); + return None; + } + } + } else if matches!(phase, Some(EpochPhase::Midpoint | EpochPhase::Late)) { + let mut reservation = self.reshare.next_log(height).await; + let payload = reservation + .as_mut() + .and_then(|reservation| reservation.take_payload()); + (payload, reservation) + } else { + (None, None) + }; + span.record("has_payload", payload.is_some()); + let proposed = self + .inner + .propose( + context, + ancestry, + Input { + parent: input, + payload, + }, + ) + .await; + if proposed.is_some() + && let Some(reservation) = log_reservation + { + reservation.included(); + } + proposed + } + + #[tracing::instrument( + name = "dkg.reshare.application.verify", + level = "info", + skip_all, + fields( + height = field::Empty, + phase = field::Empty, + has_payload = field::Empty + ) + )] + async fn verify( + &mut self, + context: (E, Self::Context), + ancestry: impl Ancestry, + ) -> bool { + let Some(tip) = ancestry.peek().cloned() else { + return self.inner.verify(context, ancestry).await; + }; + let height = tip.height(); + let phase = self.phase(height); + let tip_payload = tip.payload(); + let span = tracing::Span::current(); + span.record("height", height.traced()); + span.record("phase", field::debug(phase)); + span.record("has_payload", tip_payload.is_some()); + + if self.final_block(height) { + match self.reshare.epoch_info(ancestry.clone()).await { + EpochInfoResponse::Available(derived) => { + if derived != tip_payload { + debug!("verification rejected: final block payload mismatch"); + return false; + } + } + EpochInfoResponse::Pending => { + debug!("verification pending: final block epoch info is not ready"); + future::pending::<()>().await; + unreachable!("pending future must not resolve"); + } + EpochInfoResponse::Following => { + debug!("verification delegated: follower has no final block epoch info"); + } + EpochInfoResponse::Unavailable => { + debug!("verification rejected: final block epoch info is unavailable"); + return false; + } + } + } else if matches!(phase, Some(EpochPhase::Early)) && tip_payload.is_some() { + // Dealer logs are only posted from the midpoint onward, so an early + // block must not carry a reshare payload. + debug!("verification rejected: early block carried reshare payload"); + return false; + } + self.inner.verify(context, ancestry).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dkg::{ + reshare::{LogReservation, Message}, + tests::mocks::{self, TestBlock, TestBlsVariant, TestContext, TestScheme}, + types::{EpochInfo, EpochOutcome}, + }; + use commonware_actor::mailbox; + use commonware_consensus::{ + CertifiableBlock, Heightable, + marshal::ancestry, + types::{Epoch, Height, Round, View}, + }; + use commonware_cryptography::{ + Digestible, Signer, + bls12381::{ + dkg::feldman_desmedt::deal, + primitives::{sharing::Mode, variant::MinPk}, + }, + ed25519::{PrivateKey, PublicKey}, + sha256::Sha256, + }; + use commonware_runtime::{Clock, Metrics, Runner, Spawner, Supervisor, deterministic}; + use commonware_utils::{ + Acknowledgement, N3f1, NZU32, NZU64, NZUsize, TestRng, channel::oneshot, ordered::Set, + sync::Mutex, + }; + use futures::{ + FutureExt, + future::{Either, select}, + pin_mut, + }; + use rand_core::Rng; + use std::{sync::Arc, time::Duration}; + + type TestPayload = Payload; + type TestResponse = EpochInfoResponse; + type TestWrapper = Application; + + impl CertifiableBlock for TestBlock { + type Context = TestContext; + + fn context(&self) -> Self::Context { + self.context().clone() + } + } + + #[derive(Clone, Copy, PartialEq, Eq)] + enum ProposalBehavior { + Accept, + Reject, + Pending, + } + + #[derive(Clone)] + struct RecordingApp { + proposed: Arc>>>, + proposal_behavior: ProposalBehavior, + proposal_entered: Arc>>>, + verify_count: Arc>, + verify_result: bool, + } + + impl RecordingApp { + fn accepting() -> Self { + Self { + proposed: Arc::new(Mutex::new(Vec::new())), + proposal_behavior: ProposalBehavior::Accept, + proposal_entered: Arc::new(Mutex::new(None)), + verify_count: Arc::new(Mutex::new(0)), + verify_result: true, + } + } + + fn rejecting() -> Self { + Self { + proposal_behavior: ProposalBehavior::Reject, + ..Self::accepting() + } + } + + fn pending(proposal_entered: oneshot::Sender<()>) -> Self { + Self { + proposal_behavior: ProposalBehavior::Pending, + proposal_entered: Arc::new(Mutex::new(Some(proposal_entered))), + ..Self::accepting() + } + } + + fn proposed(&self) -> Vec> { + self.proposed.lock().clone() + } + + fn verify_count(&self) -> usize { + *self.verify_count.lock() + } + } + + impl ConsensusApplication for RecordingApp + where + E: Rng + Spawner + Metrics + Clock, + { + type SigningScheme = TestScheme; + type Context = TestContext; + type Block = TestBlock; + type Input = Input<(), TestBlsVariant, PrivateKey>; + + async fn propose( + &mut self, + (_, context): (E, Self::Context), + ancestry: impl Ancestry, + input: Self::Input, + ) -> Option { + let parent = ancestry.peek()?.clone(); + self.proposed.lock().push(input.payload.clone()); + if let Some(entered) = self.proposal_entered.lock().take() { + let _ = entered.send(()); + } + + if self.proposal_behavior == ProposalBehavior::Reject { + return None; + } + + if self.proposal_behavior == ProposalBehavior::Pending { + future::pending().await + } + + let block = + TestBlock::new::(context, parent.digest(), parent.height().next(), 0); + Some(match input.payload { + Some(payload) => { + block.with_payload::(NZU32!(16), payload) + } + None => block, + }) + } + + async fn verify(&mut self, _: (E, Self::Context), _: impl Ancestry) -> bool { + *self.verify_count.lock() += 1; + self.verify_result + } + } + + fn wrapper(context: &deterministic::Context, response: TestResponse) -> TestWrapper { + wrapper_with_inner(context, response, RecordingApp::accepting()) + } + + fn wrapper_with_inner( + context: &deterministic::Context, + response: TestResponse, + inner: RecordingApp, + ) -> TestWrapper { + let (sender, mut receiver) = mailbox::new::>( + context.child("mailbox"), + NZUsize!(1), + ); + context.child("fake_actor").spawn(|_| async move { + let Some(Message::EpochInfo { + response: reply, .. + }) = receiver.recv().await + else { + return; + }; + let _ = reply.send(response); + }); + + Application::new(inner, Mailbox::new(sender), NZU64!(2)) + } + + fn log_wrapper( + context: &deterministic::Context, + payload: TestPayload, + inner: RecordingApp, + ) -> (TestWrapper, oneshot::Receiver) { + let (sender, mut receiver) = mailbox::new::>( + context.child("mailbox"), + NZUsize!(4), + ); + let (release_tx, release_rx) = oneshot::channel(); + context.child("fake_actor").spawn(|_| async move { + let mut served_at = None; + let mut release_tx = Some(release_tx); + while let Some(message) = receiver.recv().await { + match message { + Message::NextLog { + height, + release, + response, + .. + } => { + let reservation = served_at.is_none().then(|| { + served_at = Some(height); + LogReservation::new(height, payload.clone(), release) + }); + let _ = response.send(reservation); + } + Message::ReleaseLog { height } => { + if served_at == Some(height) { + served_at = None; + } + if let Some(release_tx) = release_tx.take() { + let _ = release_tx.send(height); + } + } + Message::EpochInfo { response, .. } => { + let _ = response.send(EpochInfoResponse::Unavailable); + } + Message::Finalized { response, .. } => { + response.acknowledge(); + } + } + } + }); + + ( + Application::new(inner, Mailbox::new(sender), NZU64!(4)), + release_rx, + ) + } + + fn leader() -> PrivateKey { + PrivateKey::from_seed(99) + } + + fn block_context(parent: &TestBlock, view: u64) -> TestContext { + TestContext { + round: Round::new(Epoch::zero(), View::new(view)), + leader: leader().public_key(), + parent: (View::zero(), parent.digest()), + } + } + + fn signers() -> Vec { + (0..4).map(PrivateKey::from_seed).collect() + } + + fn players() -> Set { + Set::from_iter_dedup(signers().iter().map(Signer::public_key)) + } + + fn epoch_payload(seed: u64) -> TestPayload { + let (output, _) = + deal::(TestRng::new(seed), Mode::NonZeroCounter, players()) + .expect("trusted deal"); + Payload::EpochInfo(EpochInfo { + outcome: EpochOutcome::Success, + epoch: Epoch::new(1), + output, + players: Set::default(), + next_players: Set::default(), + }) + } + + fn final_block(parent: &TestBlock, payload: Option) -> Arc { + let block = TestBlock::new::( + block_context(parent, 1), + parent.digest(), + parent.height().next(), + 0, + ); + let block = match payload { + Some(payload) => { + block.with_payload::(NZU32!(16), payload) + } + None => block, + }; + Arc::new(block) + } + + fn midpoint_parent() -> TestBlock { + let genesis = mocks::genesis_block(leader().public_key()); + TestBlock::new::( + block_context(&genesis, 1), + genesis.digest(), + genesis.height().next(), + 0, + ) + } + + #[test] + fn proposal_none_releases_dealer_log_reservation() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = midpoint_parent(); + let payload = epoch_payload(10); + let inner = RecordingApp::rejecting(); + let (mut app, release_rx) = log_wrapper(&context, payload.clone(), inner.clone()); + + let proposed = app + .propose( + (context.child("app"), block_context(&parent, 2)), + ancestry::from_iter([Arc::new(parent.clone())]), + (), + ) + .await; + assert!(proposed.is_none()); + assert_eq!( + release_rx.await.expect("reservation should be released"), + Height::new(2) + ); + + let proposed = app + .propose( + (context.child("app_retry"), block_context(&parent, 3)), + ancestry::from_iter([Arc::new(parent)]), + (), + ) + .await; + assert!(proposed.is_none()); + let proposed_payloads = inner.proposed(); + assert_eq!(proposed_payloads.len(), 2); + assert!(proposed_payloads[0] == Some(payload.clone())); + assert!(proposed_payloads[1] == Some(payload)); + }); + } + + #[test] + fn dropped_proposal_future_releases_dealer_log_reservation() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = midpoint_parent(); + let payload = epoch_payload(11); + let (entered_tx, entered_rx) = oneshot::channel(); + let mut entered_rx = entered_rx; + let inner = RecordingApp::pending(entered_tx); + let (sender, mut receiver) = mailbox::new::< + Message, + >(context.child("mailbox"), NZUsize!(4)); + let mut app = Application::new(inner.clone(), Mailbox::new(sender), NZU64!(4)); + + let mut propose = Box::pin(app.propose( + (context.child("app"), block_context(&parent, 2)), + ancestry::from_iter([Arc::new(parent)]), + (), + )); + assert!(propose.as_mut().now_or_never().is_none()); + + let Some(Message::NextLog { + height, + release, + response, + .. + }) = receiver.recv().await + else { + panic!("proposal should request a dealer log"); + }; + assert_eq!(height, Height::new(2)); + let reservation = LogReservation::new(height, payload.clone(), release); + assert!( + response.send(Some(reservation)).is_ok(), + "proposal should still be waiting for log" + ); + + assert!(propose.as_mut().now_or_never().is_none()); + entered_rx + .try_recv() + .expect("proposal should enter inner application"); + drop(propose); + + let Some(Message::ReleaseLog { height }) = receiver.recv().await else { + panic!("dropped proposal should release reservation"); + }; + assert_eq!(height, Height::new(2)); + + if let Ok(Message::ReleaseLog { height }) = receiver.try_recv() { + panic!("reservation released more than once at {height:?}"); + } + let proposed_payloads = inner.proposed(); + assert_eq!(proposed_payloads.len(), 1); + assert!(proposed_payloads[0] == Some(payload)); + }); + } + + #[test] + fn successful_proposal_keeps_dealer_log_reserved() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = midpoint_parent(); + let payload = epoch_payload(12); + let inner = RecordingApp::accepting(); + let (mut app, release_rx) = log_wrapper(&context, payload.clone(), inner.clone()); + + let proposed = app + .propose( + (context.child("app"), block_context(&parent, 2)), + ancestry::from_iter([Arc::new(parent)]), + (), + ) + .await + .expect("proposal should be built"); + assert!(proposed.payload() == Some(payload.clone())); + let proposed_payloads = inner.proposed(); + assert_eq!(proposed_payloads.len(), 1); + assert!(proposed_payloads[0] == Some(payload)); + + let timeout = context.sleep(Duration::from_millis(1)); + pin_mut!(release_rx); + pin_mut!(timeout); + match select(release_rx, timeout).await { + Either::Left((released, _)) => { + panic!("successful proposal released reservation: {released:?}"); + } + Either::Right(((), _)) => {} + } + }); + } + + #[test] + fn proposal_skips_unavailable_final_epoch_info() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = mocks::genesis_block(leader().public_key()); + let mut app = wrapper(&context, EpochInfoResponse::Unavailable); + let inner = app.inner.clone(); + + let proposed = app + .propose( + (context.child("app"), block_context(&parent, 1)), + ancestry::from_iter([Arc::new(parent)]), + (), + ) + .await; + + assert!(proposed.is_none()); + assert!(inner.proposed().is_empty()); + }); + } + + #[test] + fn proposal_preserves_legitimate_no_artifact_final_block() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = mocks::genesis_block(leader().public_key()); + let mut app = wrapper(&context, EpochInfoResponse::Available(None)); + let inner = app.inner.clone(); + + let proposed = app + .propose( + (context.child("app"), block_context(&parent, 1)), + ancestry::from_iter([Arc::new(parent)]), + (), + ) + .await + .expect("proposal should be built"); + + assert!(proposed.payload().is_none()); + let proposed_payloads = inner.proposed(); + assert_eq!(proposed_payloads.len(), 1); + assert!(proposed_payloads[0].is_none()); + }); + } + + #[test] + fn proposal_includes_available_final_epoch_info() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = mocks::genesis_block(leader().public_key()); + let payload = epoch_payload(7); + let mut app = wrapper( + &context, + EpochInfoResponse::Available(Some(payload.clone())), + ); + let inner = app.inner.clone(); + + let proposed = app + .propose( + (context.child("app"), block_context(&parent, 1)), + ancestry::from_iter([Arc::new(parent)]), + (), + ) + .await + .expect("proposal should be built"); + + assert!(proposed.payload() == Some(payload.clone())); + let proposed_payloads = inner.proposed(); + assert_eq!(proposed_payloads.len(), 1); + assert!(proposed_payloads[0] == Some(payload)); + }); + } + + #[test] + fn verification_rejects_unavailable_final_epoch_info() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = Arc::new(mocks::genesis_block(leader().public_key())); + let tip = final_block(&parent, Some(epoch_payload(1))); + let mut app = wrapper(&context, EpochInfoResponse::Unavailable); + let inner = app.inner.clone(); + + let verified = app + .verify( + (context.child("app"), block_context(&parent, 1)), + ancestry::from_iter([tip, parent]), + ) + .await; + + assert!(!verified); + assert_eq!(inner.verify_count(), 0); + }); + } + + #[test] + fn verification_delegates_when_following_without_epoch_info() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + for expected in [true, false] { + let parent = Arc::new(mocks::genesis_block(leader().public_key())); + let tip = final_block(&parent, Some(epoch_payload(1))); + let mut app = wrapper(&context, EpochInfoResponse::Following); + app.inner.verify_result = expected; + let inner = app.inner.clone(); + + let verified = app + .verify( + (context.child("app"), block_context(&parent, 1)), + ancestry::from_iter([tip, parent]), + ) + .await; + + assert_eq!(verified, expected); + assert_eq!(inner.verify_count(), 1); + } + }); + } + + #[test] + fn verification_stays_pending_when_final_epoch_info_not_ready() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = Arc::new(mocks::genesis_block(leader().public_key())); + let tip = final_block(&parent, Some(epoch_payload(1))); + let mut app = wrapper(&context, EpochInfoResponse::Pending); + let inner = app.inner.clone(); + + let verify = app.verify( + (context.child("app"), block_context(&parent, 1)), + ancestry::from_iter([tip, parent]), + ); + let timeout = context.sleep(Duration::from_millis(1)); + pin_mut!(verify); + pin_mut!(timeout); + + match select(verify, timeout).await { + Either::Left((verified, _)) => { + panic!("verification resolved before epoch info was ready: {verified}"); + } + Either::Right(((), _)) => {} + } + assert_eq!(inner.verify_count(), 0); + }); + } + + #[test] + fn verification_accepts_legitimate_no_artifact_final_block() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = Arc::new(mocks::genesis_block(leader().public_key())); + let tip = final_block(&parent, None); + let mut app = wrapper(&context, EpochInfoResponse::Available(None)); + let inner = app.inner.clone(); + + let verified = app + .verify( + (context.child("app"), block_context(&parent, 1)), + ancestry::from_iter([tip, parent]), + ) + .await; + + assert!(verified); + assert_eq!(inner.verify_count(), 1); + }); + } + + #[test] + fn verification_accepts_equal_final_epoch_info() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = Arc::new(mocks::genesis_block(leader().public_key())); + let payload = epoch_payload(2); + let tip = final_block(&parent, Some(payload.clone())); + let mut app = wrapper(&context, EpochInfoResponse::Available(Some(payload))); + let inner = app.inner.clone(); + + let verified = app + .verify( + (context.child("app"), block_context(&parent, 1)), + ancestry::from_iter([tip, parent]), + ) + .await; + + assert!(verified); + assert_eq!(inner.verify_count(), 1); + }); + } + + #[test] + fn verification_rejects_mismatched_final_epoch_info() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let parent = Arc::new(mocks::genesis_block(leader().public_key())); + let tip = final_block(&parent, Some(epoch_payload(3))); + let response = EpochInfoResponse::Available(Some(epoch_payload(4))); + let mut app = wrapper(&context, response); + let inner = app.inner.clone(); + + let verified = app + .verify( + (context.child("app"), block_context(&parent, 1)), + ancestry::from_iter([tip, parent]), + ) + .await; + + assert!(!verified); + assert_eq!(inner.verify_count(), 0); + }); + } +} diff --git a/glue/src/dkg/reshare/mailbox.rs b/glue/src/dkg/reshare/mailbox.rs new file mode 100644 index 00000000000..ec1c473cd9c --- /dev/null +++ b/glue/src/dkg/reshare/mailbox.rs @@ -0,0 +1,326 @@ +//! Reshare [`Actor`] ingress. +//! +//! [`Actor`]: super::Actor + +use crate::dkg::{ReshareBlock, types::Payload}; +use commonware_actor::{ + Feedback, + mailbox::{Policy, Sender as ActorSender}, +}; +use commonware_consensus::{Reporter, marshal::Update, types::Height}; +use commonware_cryptography::{Signer, bls12381::primitives::variant::Variant}; +use commonware_runtime::telemetry::traces::TracedExt as _; +use commonware_utils::{Acknowledgement, acknowledgement::Exact, channel::oneshot}; +use futures::Stream; +use std::{collections::VecDeque, pin::Pin, sync::Arc}; +use tracing::{Span, error, info_span}; + +/// Type-erased block ancestry stream sent through the actor mailbox. +pub(crate) type ErasedAncestry = Pin> + Send>>; + +/// Response to a final-block epoch artifact request. +#[derive(Clone, PartialEq, Eq)] +pub enum EpochInfoResponse +where + V: Variant, + C: Signer, +{ + /// The actor derived a stable response. + /// + /// `None` is a legitimate response only for a failed one-shot DKG final + /// block, which intentionally carries no epoch artifact. + Available(Option>), + /// The actor cannot answer this request yet. + /// + /// This is not evidence that a proposed artifact is invalid. Verification + /// remains pending until the request is canceled or local progress catches up. + Pending, + /// The actor is following the epoch without its protocol history. + /// + /// It cannot derive the artifact, but that absence is not evidence that a + /// proposed artifact is invalid. + Following, + /// The actor was expected to derive the artifact but cannot produce it. + Unavailable, +} + +/// A dealer log reserved for one proposal attempt. +/// +/// Dropping the reservation releases the log back to the reshare actor. Call +/// [`included`](Self::included) only after the wrapped application returns a +/// block for the proposal attempt that received this payload. +#[must_use = "dropping a log reservation releases it for another proposal"] +pub struct LogReservation +where + B: ReshareBlock, + V: Variant, + C: Signer, + A: Acknowledgement, +{ + height: Height, + payload: Option>, + release: Option>>, +} + +impl LogReservation +where + B: ReshareBlock, + V: Variant, + C: Signer, + A: Acknowledgement, +{ + pub(crate) const fn new( + height: Height, + payload: Payload, + release: ActorSender>, + ) -> Self { + Self { + height, + payload: Some(payload), + release: Some(release), + } + } + + /// Takes the reserved dealer log payload. + /// + /// Returns `None` if the payload was already taken. + pub const fn take_payload(&mut self) -> Option> { + self.payload.take() + } + + /// Keeps the log reserved for this height until finalization confirms + /// whether the proposal landed on-chain. + pub fn included(mut self) { + self.release = None; + } +} + +impl Drop for LogReservation +where + B: ReshareBlock, + V: Variant, + C: Signer, + A: Acknowledgement, +{ + fn drop(&mut self) { + let Some(release) = self.release.take() else { + return; + }; + let _ = release.enqueue(Message::ReleaseLog { + height: self.height, + }); + } +} + +/// A message that can be sent to the [`Actor`]. +/// +/// [`Actor`]: super::Actor +#[allow(clippy::large_enum_variant)] +pub enum Message +where + B: ReshareBlock, + V: Variant, + C: Signer, + A: Acknowledgement, +{ + /// A request for the next finalized dealer log to include before the final + /// block of the epoch. + /// + /// `height` is the height of the block being proposed. The actor uses it to + /// avoid re-offering a log into competing proposals while one it already + /// served into may still finalize. + NextLog { + span: Span, + height: Height, + release: ActorSender, + response: oneshot::Sender>>, + }, + + /// A proposal attempt was canceled or returned no block after receiving a + /// dealer log. + ReleaseLog { height: Height }, + + /// A request for the final block's speculative [`EpochInfo`](crate::dkg::types::EpochInfo). + EpochInfo { + span: Span, + ancestry: ErasedAncestry, + response: oneshot::Sender>, + }, + + /// A new block has been finalized. + Finalized { + span: Span, + block: Arc, + response: A, + }, +} + +impl Message +where + B: ReshareBlock, + V: Variant, + C: Signer, + A: Acknowledgement, +{ + fn response_closed(&self) -> bool { + match self { + Self::NextLog { response, .. } => response.is_closed(), + Self::ReleaseLog { .. } => false, + Self::EpochInfo { response, .. } => response.is_closed(), + Self::Finalized { .. } => false, + } + } +} + +impl Policy for Message +where + B: ReshareBlock, + V: Variant, + C: Signer, + A: Acknowledgement, +{ + type Overflow = VecDeque; + + fn handle(overflow: &mut VecDeque, message: Self) { + if message.response_closed() { + return; + } + overflow.push_back(message); + } +} + +/// Inbox for sending messages to the reshare [`Actor`]. +/// +/// [`Actor`]: super::Actor +#[derive(Clone)] +pub struct Mailbox +where + B: ReshareBlock, + V: Variant, + C: Signer, + A: Acknowledgement, +{ + sender: ActorSender>, +} + +impl Mailbox +where + B: ReshareBlock, + V: Variant, + C: Signer, + A: Acknowledgement, +{ + /// Create a new mailbox. + pub const fn new(sender: ActorSender>) -> Self { + Self { sender } + } + + /// Request a dealer log for inclusion before the final block of the epoch. + /// + /// `height` is the height of the block being proposed. + pub async fn next_log(&mut self, height: Height) -> Option> { + let (response_tx, response_rx) = oneshot::channel(); + let span = info_span!("dkg.reshare.mailbox.next_log", height = height.traced()); + if !self + .sender + .enqueue(Message::NextLog { + span, + height, + release: self.sender.clone(), + response: response_tx, + }) + .accepted() + { + error!("failed to send request for next dealer log"); + return None; + } + + match response_rx.await { + Ok(outcome) => outcome, + Err(err) => { + error!(?err, "failed to receive payload response"); + None + } + } + } + + /// Request the final block's next-epoch artifact. + pub async fn epoch_info( + &mut self, + ancestry: impl Stream> + Send + 'static, + ) -> EpochInfoResponse { + let (response_tx, response_rx) = oneshot::channel(); + let span = info_span!("dkg.reshare.mailbox.epoch_info"); + if !self + .sender + .enqueue(Message::EpochInfo { + span, + ancestry: Box::pin(ancestry), + response: response_tx, + }) + .accepted() + { + error!("failed to send request for epoch info"); + return EpochInfoResponse::Unavailable; + } + + match response_rx.await { + Ok(outcome) => outcome, + Err(err) => { + error!(?err, "failed to receive epoch info response"); + EpochInfoResponse::Unavailable + } + } + } +} + +impl Reporter for Mailbox +where + B: ReshareBlock, + V: Variant, + C: Signer, + A: Acknowledgement, +{ + type Activity = Update; + + fn report(&mut self, update: Self::Activity) -> Feedback { + let Update::Block(block, ack_tx) = update else { + return Feedback::Ok; + }; + let span = info_span!( + "dkg.reshare.mailbox.finalized", + height = block.height().traced(), + digest = %block.digest() + ); + self.sender.enqueue(Message::Finalized { + span, + block, + response: ack_tx, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dkg::tests::mocks::{TestBlock, TestBlsVariant}; + use commonware_actor::mailbox; + use commonware_cryptography::ed25519::PrivateKey; + use commonware_runtime::{Runner, deterministic}; + use commonware_utils::NZUsize; + + type TestMessage = Message; + + #[test] + fn next_log_returns_none_when_actor_gone() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let (sender, receiver) = mailbox::new::(context, NZUsize!(1)); + drop(receiver); + + let mut mailbox = Mailbox::::new(sender); + + assert!(mailbox.next_log(Height::new(1)).await.is_none()); + }); + } +} diff --git a/glue/src/dkg/reshare/metrics.rs b/glue/src/dkg/reshare/metrics.rs new file mode 100644 index 00000000000..2f5bc734d3b --- /dev/null +++ b/glue/src/dkg/reshare/metrics.rs @@ -0,0 +1,118 @@ +//! Metrics for the reshare actor. + +use commonware_cryptography::{ + PublicKey, + bls12381::{dkg::feldman_desmedt::Output, primitives::variant::Variant}, +}; +use commonware_runtime::{ + Metrics as RuntimeMetrics, + telemetry::metrics::{Counter, EncodeStruct, Gauge, GaugeExt, GaugeFamily, MetricsExt as _}, +}; + +/// Per-peer metric label. +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeStruct)] +pub(crate) struct Peer { + peer: P, +} + +impl From

for Peer

{ + fn from(peer: P) -> Self { + Self { peer } + } +} + +/// Reshare actor phase. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Phase { + /// Observing the epoch without participating. + Following = 0, + /// Loading state and participant sets. + Setup = 1, + /// Exchanging dealer messages and player acknowledgments. + Dealing = 2, + /// Including signed dealer logs and committing the next epoch when the + /// final block finalizes. + Inclusion = 3, +} + +/// Metrics for the reshare actor. +pub(crate) struct Metrics { + /// Number of successful epochs. + pub(crate) successful_epochs: Counter, + /// Number of failed epochs. + pub(crate) failed_epochs: Counter, + /// Number of epochs where our share was revealed. + pub(crate) our_reveals: Counter, + /// Total revealed shares across successful epochs. + pub(crate) all_reveals: Counter, + /// Latest epoch where a valid share was received from a dealer. + pub(crate) latest_share: GaugeFamily>, + /// Latest epoch where a valid ack was received from a player. + pub(crate) latest_ack: GaugeFamily>, + /// Current DKG epoch. + pub(crate) current_epoch: Gauge, + /// Current DKG round. + pub(crate) current_round: Gauge, + /// Current actor phase. + pub(crate) phase: Gauge, +} + +impl Metrics

{ + /// Registers reshare metrics in `context`. + pub(crate) fn new(context: &E) -> Self { + let current_epoch = context.gauge("current_epoch", "Current DKG epoch"); + let _ = current_epoch.try_set(0); + + let current_round = context.gauge("current_round", "Current DKG round"); + let _ = current_round.try_set(0); + + let phase = context.gauge("phase", "Current reshare actor phase"); + let _ = phase.try_set(Phase::Following as i64); + + Self { + successful_epochs: context.counter("successful_epochs", "Successful epochs"), + failed_epochs: context.counter("failed_epochs", "Failed epochs"), + our_reveals: context.counter("our_reveals", "Our share was revealed"), + all_reveals: context.counter("all_reveals", "All share reveals"), + latest_share: context.family( + "latest_share", + "Epoch of latest valid share received per dealer", + ), + latest_ack: context.family( + "latest_ack", + "Epoch of latest valid ack received per player", + ), + current_epoch, + current_round, + phase, + } + } + + /// Records the current phase. + pub(crate) fn set_phase(&self, phase: Phase) { + let _ = self.phase.try_set(phase as i64); + } + + /// Records the latest epoch where a valid share was received from `dealer`. + pub(crate) fn record_share(&self, dealer: &P, epoch: u64) { + let peer = Peer::from(dealer.clone()); + let _ = self.latest_share.get_or_create(&peer).try_set_max(epoch); + } + + /// Records the latest epoch where a valid ack was received from `player`. + pub(crate) fn record_ack(&self, player: &P, epoch: u64) { + let peer = Peer::from(player.clone()); + let _ = self.latest_ack.get_or_create(&peer).try_set_max(epoch); + } + + /// Records a successful ceremony and the shares it revealed, noting whether + /// our own share (`ours`) was among them. + pub(crate) fn record_success(&self, output: &Output, ours: &P) { + self.successful_epochs.inc(); + let revealed = output.revealed(); + self.all_reveals.inc_by(revealed.len() as u64); + if revealed.position(ours).is_some() { + self.our_reveals.inc(); + } + } +} diff --git a/glue/src/dkg/reshare/mod.rs b/glue/src/dkg/reshare/mod.rs new file mode 100644 index 00000000000..d17a36134f2 --- /dev/null +++ b/glue/src/dkg/reshare/mod.rs @@ -0,0 +1,193 @@ +//! Continuous BLS threshold-key resharing for an application chain. +//! +//! This module runs the ongoing reshare protocol after a chain already has an +//! initial threshold output. It lets an application rotate the set of threshold +//! share holders over time without exposing the aggregate signing key and +//! without requiring a trusted party to redistribute private shares. +//! +//! The reshare actor is a protocol companion that: +//! +//! - reads finalized [`EpochInfo`](crate::dkg::types::EpochInfo) artifacts from +//! application blocks, +//! - exchanges private Feldman-Desmedt dealings with peers during each epoch, +//! - asks the application to include public dealer logs on-chain, +//! - derives the next epoch's [`EpochInfo`](crate::dkg::types::EpochInfo), and +//! - registers signer or verifier schemes through the application-provided +//! [`Registrar`](crate::dkg::Registrar). +//! +//! # Epoch Artifacts +//! +//! Every epoch is described by an [`EpochInfo`](crate::dkg::types::EpochInfo) +//! carried in a finalized boundary block. For epoch zero this artifact is part +//! of genesis. For later epochs it is carried in the final block of the previous +//! epoch. +//! +//! An epoch artifact is a lookahead: +//! +//! - `output` is the public threshold output whose players are the dealers for +//! the described epoch. +//! - `players` are the share holders targeted by the ceremony in the described +//! epoch. +//! - `next_players` are announced one epoch early so future players can connect +//! and state sync before they must receive private dealings. +//! - `outcome` records whether the ceremony that produced this boundary +//! artifact succeeded or failed. +//! +//! On success, the artifact contains the newly generated output. On failure, +//! the artifact carries the previous output forward, advances `players` to the +//! previously announced `next_players`, and refreshes `next_players` from the +//! [`ParticipantsProvider`](crate::dkg::ParticipantsProvider). +//! +//! # Protocol Flow +//! +//! Each epoch has three logical windows: +//! +//! 1. **Setup** loads the finalized boundary artifact, recovers durable protocol +//! state, registers the current epoch's scheme, and determines whether this +//! node is a dealer, player, both, or only an observer. +//! 2. **Dealing** runs in the early half of the epoch. Dealers send private +//! shares directly to players over the DKG P2P channel. Players verify those +//! shares and return signed acknowledgements. +//! 3. **Inclusion** runs from the midpoint through the final block. Dealers with +//! enough acknowledgements construct public dealer logs. The application +//! includes those logs in blocks, and the final block carries the next +//! epoch's [`EpochInfo`](crate::dkg::types::EpochInfo). +//! +//! ```text +//! boundary EpochInfo(E) +//! | +//! v +//! setup and scheme registration +//! | +//! v +//! early epoch: private dealings and acknowledgements over P2P +//! | +//! v +//! midpoint onward: dealer logs are posted on-chain +//! | +//! v +//! final block: EpochInfo(E + 1) +//! ``` +//! +//! Finalized application blocks are the source of truth. Private P2P traffic may +//! be retried or recovered locally, but dealer logs and epoch artifacts affect +//! durable protocol state only after they are finalized on-chain. +//! +//! # Application Contract +//! +//! Application blocks implement [`ReshareBlock`](crate::dkg::ReshareBlock) and +//! carry at most one [`Payload`](crate::dkg::types::Payload). The application is +//! responsible for wiring proposal and verification to [`Mailbox`]: +//! +//! - Before the final block of an epoch, proposers call [`Mailbox::next_log`], +//! include the reserved dealer log if one is returned, and keep the +//! reservation only after a block is built. +//! - Before the final block of an epoch, verifiers treat dealer logs as ordinary +//! optional payloads and rely on finalized delivery to update the reshare +//! actor. +//! - At the final block of an epoch, proposers call [`Mailbox::epoch_info`] and +//! include the returned [`EpochInfo`](crate::dkg::types::EpochInfo). +//! - At the final block of an epoch, verifiers also call [`Mailbox::epoch_info`] +//! and must reject any block whose payload is not the same +//! [`EpochInfo`](crate::dkg::types::EpochInfo). +//! +//! The final-block call receives the pending ancestry between the finalized tip +//! and the block under construction or verification. This matters because the +//! application may be proposing or verifying above the finalized tip; dealer logs +//! in that pending ancestry can change the ceremony outcome, but they must not be +//! written durably until the corresponding blocks finalize. +//! +//! Marshal must report finalized blocks to the reshare actor. The actor +//! acknowledges a finalized block only after any protocol state, secret state, +//! registrar update, and epoch fence update required by that block is complete. +//! +//! # Secret Material +//! +//! The protocol deliberately does not prescribe secret storage. Applications +//! provide a [`SecretStore`](crate::dkg::SecretStore) that matches their security +//! policy. +//! +//! The store contains private shares, private dealings, and dealer randomness +//! seeds. These values must not be placed in public protocol storage or +//! application state. The actor uses them for restart recovery, including +//! carrying a valid share forward when a ceremony fails and the previous +//! threshold output remains active. +//! +//! # Offline Players +//! +//! A validator that is selected as a `player` must be online and reachable +//! during the early dealing window if it expects its new secret share to remain +//! private. +//! +//! Feldman-Desmedt resharing preserves liveness by allowing dealers to publish +//! reveal evidence for players that do not return valid acknowledgements. If a +//! validator is offline while it is a `player`, the ceremony can still succeed, +//! but the validator's secret share for the new output will be revealed in the +//! public dealer logs. The resulting output is protocol-valid; reveal-bearing +//! outputs are not rejected by this reshare actor. +//! +//! Operationally, an offline player should treat the affected secret share as +//! public. It must not assume that coming back online later restores the +//! privacy of that share. Applications that require every active signing share +//! to remain unrevealed must enforce that policy outside this protocol. +//! +//! # State Sync +//! +//! Reshare is compatible with application state sync, but timing matters. A node +//! that joins through state sync can participate in epoch `E` only if it was +//! already announced as a `next_player` in epoch `E - 1`. That announcement gives +//! the node time to discover the boundary artifact, state sync, start consensus, +//! and be online for the early private dealings of epoch `E`. +//! +//! If a node state syncs during the epoch in which it is already a `player`, it +//! has missed non-replayable private dealings. The actor enters follower mode for +//! that partial epoch and will not participate in the ceremony. If the ceremony +//! needs that node's share, the share will be revealed as part of the public +//! outcome. +//! +//! This is why [`ParticipantsProvider`](crate::dkg::ParticipantsProvider) can be +//! backed by chain state: the chain announces future players one epoch before +//! their shares are needed. +//! +//! # One-Shot DKG +//! +//! Initial threshold-secret generation is exposed through +//! [`bootstrap`](crate::dkg::bootstrap). That engine reuses the same actor in a +//! crate-private DKG mode, runs it on a contained one-epoch consensus chain, and +//! returns an [`EpochInfo`](crate::dkg::types::EpochInfo) suitable for the +//! genesis artifact of a later reshare-enabled application chain. + +use commonware_cryptography::bls12381::primitives::sharing::ModeVersion; + +/// The max supported [`ModeVersion`] of this reshare protocol implementation. +#[cfg(not(any( + commonware_stability_BETA, + commonware_stability_GAMMA, + commonware_stability_DELTA, + commonware_stability_EPSILON, + commonware_stability_RESERVED +)))] +pub const MAX_SUPPORTED_MODE: ModeVersion = ModeVersion::v1(); + +/// The max supported [`ModeVersion`] of this reshare protocol implementation. +#[cfg(any( + commonware_stability_BETA, + commonware_stability_GAMMA, + commonware_stability_DELTA, + commonware_stability_EPSILON, + commonware_stability_RESERVED +))] +pub const MAX_SUPPORTED_MODE: ModeVersion = ModeVersion::v0(); + +mod mailbox; +pub use mailbox::{EpochInfoResponse, LogReservation, Mailbox, Message}; + +mod actor; +pub(crate) use actor::DkgConfig; +pub use actor::{Actor, Config}; + +mod application; +pub use application::{Application, Input}; + +mod metrics; +pub(crate) mod store; diff --git a/glue/src/dkg/reshare/store.rs b/glue/src/dkg/reshare/store.rs new file mode 100644 index 00000000000..826af18b3fe --- /dev/null +++ b/glue/src/dkg/reshare/store.rs @@ -0,0 +1,1095 @@ +//! DKG/reshare crash-recovery storage. +//! +//! This store exists only to recover state a restarted actor cannot otherwise +//! re-obtain. After a crash the actor does not re-receive P2P messages and +//! marshal does not re-deliver finalized blocks, so this store keeps a plaintext +//! journal of the public messages a restarted node would otherwise lose: +//! +//! - public dealer messages and player acknowledgements, so a player can rebuild +//! the acks it already emitted (its private dealings are recovered from +//! [`SecretStore`]); +//! - finalized dealer logs observed during inclusion. +//! +//! The current epoch's public state is not persisted here: it is re-derived from +//! the finalized boundary block that anchors the epoch (see the setup state), so +//! this store never caches public epoch info. Everything secret stays out of this +//! plaintext store and is held only through [`SecretStore`]: shares, private +//! dealings, and the dealer RNG seed (which seeds the dealer polynomial and so +//! reveals every share that dealer distributes). + +use crate::dkg::{SecretStore, types::EpochInfo}; +use bytes::{Buf, BufMut}; +use commonware_codec::{EncodeSize, Error as CodecError, Read, ReadExt, Write}; +use commonware_consensus::types::Epoch; +use commonware_cryptography::{ + BatchVerifier, PublicKey, Signer, + bls12381::{ + dkg::feldman_desmedt::{ + Dealer as CryptoDealer, DealerLog, DealerPrivMsg, DealerPubMsg, Error as DkgError, + Info, Logs, Output, Player as CryptoPlayer, PlayerAck, SignedDealerLog, Verdict, + }, + primitives::{group, variant::Variant}, + }, + transcript::{Summary, Transcript}, +}; +use commonware_math::algebra::Random; +use commonware_parallel::Strategy; +use commonware_runtime::{ + BufferPooler, Clock, Metrics, Storage as RuntimeStorage, buffer::paged::CacheRef, +}; +use commonware_storage::journal::segmented::variable::{Config as JournalConfig, Journal}; +use commonware_utils::{Faults, N3f1, NZU16, NZUsize}; +use futures::StreamExt; +use rand_core::CryptoRng; +use std::{ + collections::BTreeMap, + num::{NonZeroU16, NonZeroU32, NonZeroUsize}, +}; +use tracing::{debug, warn}; + +const PAGE_SIZE: NonZeroU16 = NZU16!(1 << 12); // 4 KiB +const PAGE_CACHE_CAPACITY: NonZeroUsize = NZUsize!(1 << 13); // 8 KiB +const WRITE_BUFFER: NonZeroUsize = NZUsize!(1 << 12); // 4 KiB +const READ_BUFFER: NonZeroUsize = NZUsize!(1 << 20); // 1 MiB + +enum Event { + Dealing(P, DealerPubMsg), + Ack(P, PlayerAck

), + Log(P, DealerLog), +} + +impl EncodeSize for Event { + fn encode_size(&self) -> usize { + 1 + match self { + Self::Dealing(dealer, public) => dealer.encode_size() + public.encode_size(), + Self::Ack(player, ack) => player.encode_size() + ack.encode_size(), + Self::Log(dealer, log) => dealer.encode_size() + log.encode_size(), + } + } +} + +impl Write for Event { + fn write(&self, writer: &mut impl BufMut) { + match self { + Self::Dealing(dealer, public) => { + 0u8.write(writer); + dealer.write(writer); + public.write(writer); + } + Self::Ack(player, ack) => { + 1u8.write(writer); + player.write(writer); + ack.write(writer); + } + Self::Log(dealer, log) => { + 2u8.write(writer); + dealer.write(writer); + log.write(writer); + } + } + } +} + +impl Read for Event { + type Cfg = NonZeroU32; + + fn read_cfg(reader: &mut impl Buf, cfg: &Self::Cfg) -> Result { + match u8::read(reader)? { + 0 => Ok(Self::Dealing( + ReadExt::read(reader)?, + Read::read_cfg(reader, cfg)?, + )), + 1 => Ok(Self::Ack(ReadExt::read(reader)?, ReadExt::read(reader)?)), + 2 => Ok(Self::Log( + ReadExt::read(reader)?, + Read::read_cfg(reader, cfg)?, + )), + tag => Err(CodecError::InvalidEnum(tag)), + } + } +} + +#[cfg(feature = "arbitrary")] +impl arbitrary::Arbitrary<'_> for Event +where + P: for<'a> arbitrary::Arbitrary<'a>, + DealerPubMsg: for<'a> arbitrary::Arbitrary<'a>, + DealerLog: for<'a> arbitrary::Arbitrary<'a>, + PlayerAck

: for<'a> arbitrary::Arbitrary<'a>, +{ + fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { + Ok(match u.int_in_range(0..=2)? { + 0 => Self::Dealing(u.arbitrary()?, u.arbitrary()?), + 1 => Self::Ack(u.arbitrary()?, u.arbitrary()?), + _ => Self::Log(u.arbitrary()?, u.arbitrary()?), + }) + } +} + +#[cfg(all(test, feature = "arbitrary"))] +mod conformance { + use super::*; + use commonware_codec::conformance::CodecConformance; + use commonware_cryptography::{bls12381::primitives::variant::MinSig, ed25519}; + + commonware_conformance::conformance_tests! { + CodecConformance>, + } +} + +struct EpochCache { + dealings: BTreeMap, DealerPrivMsg)>, + acks: BTreeMap>, + logs: BTreeMap>, +} + +impl Default for EpochCache { + fn default() -> Self { + Self { + dealings: BTreeMap::new(), + acks: BTreeMap::new(), + logs: BTreeMap::new(), + } + } +} + +/// DKG/reshare crash-recovery store. +/// +/// The plaintext side holds only the dealer-message, acknowledgement, and +/// finalized-log journal. The current epoch's public state is re-derived from +/// finalized boundary blocks, not cached here. All secret material (shares, +/// private dealings, and the dealer RNG seed) is held only through +/// [`SecretStore`], never in plaintext. +pub struct Store +where + E: BufferPooler + Clock + RuntimeStorage + Metrics, + SS: SecretStore, + V: Variant, + P: PublicKey, +{ + secret_store: SS, + events: Journal>, + current: Option>, + epochs: BTreeMap>, +} + +impl Store +where + E: BufferPooler + Clock + RuntimeStorage + Metrics, + SS: SecretStore, + V: Variant, + P: PublicKey, +{ + /// Initializes the store and replays durable crash-recovery state. + pub async fn init( + context: E, + partition_prefix: &str, + max_participants: NonZeroU32, + mut secret_store: SS, + ) -> Self { + let page_cache = CacheRef::from_pooler(&context, PAGE_SIZE, PAGE_CACHE_CAPACITY); + let mut events = Journal::init( + context.child("events"), + JournalConfig { + partition: format!("{partition_prefix}_events"), + compression: None, + codec_config: max_participants, + page_cache, + write_buffer: WRITE_BUFFER, + }, + ) + .await + .expect("failed to initialize reshare event journal"); + + // The current epoch is not persisted: it is re-derived from finalized + // boundary blocks by the setup state, so a restarted store starts with no + // current epoch. + let current = None; + + let mut epochs = BTreeMap::>::new(); + { + let replay = events + .replay(0, 0, READ_BUFFER) + .await + .expect("failed to replay reshare events"); + futures::pin_mut!(replay); + + while let Some(result) = replay.next().await { + let (section, _, _, event) = result.expect("failed to read reshare event"); + let epoch = Epoch::new(section); + let cache = epochs.entry(epoch).or_default(); + match event { + Event::Dealing(dealer, public) => { + let private = secret_store.get_dealing(epoch, &dealer).await; + if let Some(private) = private { + cache.dealings.insert(dealer, (public, private)); + } + } + Event::Ack(player, ack) => { + cache.acks.insert(player, ack); + } + Event::Log(dealer, log) => { + cache.logs.insert(dealer, log); + } + } + } + } + + Self { + secret_store, + events, + current, + epochs, + } + } + + /// Returns the current epoch state, if one has been entered. + pub fn current(&self) -> Option> { + self.current.clone() + } + + /// Returns the share for `epoch`, if any. + pub async fn share(&mut self, epoch: Epoch) -> Option { + self.secret_store.get_share(epoch).await + } + + /// Returns the persisted dealer RNG seed for `epoch`, if any. + /// + /// A seed exists only for an epoch this node previously entered. Reusing it + /// keeps dealer randomness identical across a restart. + pub async fn seed(&mut self, epoch: Epoch) -> Option

{ + self.secret_store.get_seed(epoch).await + } + + /// Returns the persisted dealer RNG seed for `epoch`, generating a fresh + /// random seed from `rng` if none exists. + pub(crate) async fn seed_or_random(&mut self, epoch: Epoch, rng: impl CryptoRng) -> Summary { + self.seed(epoch) + .await + .unwrap_or_else(|| Summary::random(rng)) + } + + /// Persists the dealer RNG seed for `epoch`. + pub(crate) async fn put_seed(&mut self, epoch: Epoch, rng_seed: Summary) { + self.secret_store.put_seed(epoch, rng_seed).await; + } + + /// Advances to `info`, persisting its secrets before the current epoch moves. + /// + /// The public artifact in `info` is read from the finalized boundary block, + /// not from local state, and is held only in memory. The share is persisted + /// only when it matches that finalized truth; otherwise the node continues as + /// an observer. + pub async fn commit_epoch( + &mut self, + info: EpochInfo, + rng_seed: Summary, + share: Option, + ) { + let epoch = info.epoch; + if let Some(share) = share { + self.secret_store.put_share(epoch, share).await; + } + self.secret_store.put_seed(epoch, rng_seed).await; + self.current = Some(info); + } + + /// Prunes public recovery data and secret material older than `min`. + pub async fn prune(&mut self, min: Epoch) { + self.epochs.retain(|epoch, _| *epoch >= min); + // Prune the recovery journal and the secret store concurrently; they are + // independent backends. + let events = &mut self.events; + let secret = &mut self.secret_store; + futures::join!( + async { + events + .prune(min.get()) + .await + .expect("failed to prune reshare events") + }, + secret.prune(min), + ); + } + + fn cache(&mut self, epoch: Epoch) -> &mut EpochCache { + self.epochs.entry(epoch).or_default() + } + + /// Returns finalized dealer logs for `epoch`. + pub fn logs(&self, epoch: Epoch) -> BTreeMap> { + self.epochs + .get(&epoch) + .map(|cache| cache.logs.clone()) + .unwrap_or_default() + } + + /// Returns true if `dealer` already has a finalized log recorded. + pub fn has_log(&self, epoch: Epoch, dealer: &P) -> bool { + self.epochs + .get(&epoch) + .is_some_and(|cache| cache.logs.contains_key(dealer)) + } + + fn dealings(&self, epoch: Epoch) -> Vec<(P, DealerPubMsg, DealerPrivMsg)> { + self.epochs + .get(&epoch) + .map(|cache| { + cache + .dealings + .iter() + .map(|(dealer, (public, private))| { + (dealer.clone(), public.clone(), private.clone()) + }) + .collect() + }) + .unwrap_or_default() + } + + fn acks(&self, epoch: Epoch) -> Vec<(P, PlayerAck

)> { + self.epochs + .get(&epoch) + .map(|cache| { + cache + .acks + .iter() + .map(|(player, ack)| (player.clone(), ack.clone())) + .collect() + }) + .unwrap_or_default() + } + + async fn append_dealing( + &mut self, + epoch: Epoch, + dealer: P, + public: DealerPubMsg, + private: DealerPrivMsg, + ) -> bool { + if self + .epochs + .get(&epoch) + .is_some_and(|cache| cache.dealings.contains_key(&dealer)) + { + return false; + } + // Persist the private dealing (secret store) and the public dealer message + // (recovery journal) concurrently. Both are durable before this returns, so + // the ack the caller emits next is always backed by recoverable state. A + // crash mid-write is safe: replay loads a dealing only when both its public + // and private parts survived. + let event = Event::Dealing(dealer.clone(), public.clone()); + let secret = &mut self.secret_store; + let events = &mut self.events; + futures::join!( + secret.put_dealing(epoch, dealer.clone(), private.clone()), + append_synced(events, epoch, &event), + ); + self.cache(epoch).dealings.insert(dealer, (public, private)); + true + } + + async fn append_ack(&mut self, epoch: Epoch, player: P, ack: PlayerAck

) -> bool { + if self + .epochs + .get(&epoch) + .is_some_and(|cache| cache.acks.contains_key(&player)) + { + return false; + } + let event = Event::Ack(player.clone(), ack.clone()); + append_synced(&mut self.events, epoch, &event).await; + self.cache(epoch).acks.insert(player, ack); + true + } + + /// Records a finalized dealer log. + pub async fn append_log(&mut self, epoch: Epoch, dealer: P, log: DealerLog) -> bool { + if self.has_log(epoch, &dealer) { + return false; + } + let event = Event::Log(dealer.clone(), log.clone()); + append_synced(&mut self.events, epoch, &event).await; + self.cache(epoch).logs.insert(dealer, log); + true + } + + /// Replays dealer state for `epoch`. + pub fn create_dealer( + &self, + epoch: Epoch, + signer: C, + info: Info, + share: Option, + rng_seed: Summary, + ) -> Option> + where + C: Signer, + M: Faults, + { + if self.has_log(epoch, &signer.public_key()) { + return None; + } + let (mut dealer, public, private) = CryptoDealer::start::( + Transcript::resume(rng_seed).noise(b"dealer-rng"), + info, + signer, + share, + ) + .expect("failed to create reshare dealer"); + let mut unsent: BTreeMap = private.into_iter().collect(); + for (player, ack) in self.acks(epoch) { + if unsent.contains_key(&player) + && dealer.receive_player_ack(player.clone(), ack).is_ok() + { + unsent.remove(&player); + debug!(?epoch, ?player, "replayed reshare ack"); + } + } + Some(Dealer { + dealer: Some(dealer), + public, + unsent, + finalized: None, + }) + } + + /// Replays player state for `epoch`. + /// + /// Returns `None` when this node observes an on-chain dealer log carrying its + /// own acknowledgement but has lost the matching private dealing; see + /// [`resume_player`](Self::resume_player). + pub fn create_player( + &self, + epoch: Epoch, + signer: C, + info: Info, + ) -> Option> + where + C: Signer, + M: Faults, + { + self.resume_player::(epoch, signer, info, &self.logs(epoch)) + } + + /// Replays player state using a supplied, non-durable log view. + /// + /// Returns `None` under the same missing-dealing condition as + /// [`create_player`](Self::create_player). + pub fn create_player_with_logs( + &self, + epoch: Epoch, + signer: C, + info: Info, + logs: &BTreeMap>, + ) -> Option> + where + C: Signer, + M: Faults, + { + self.resume_player::(epoch, signer, info, logs) + } + + /// Resumes player state for `epoch` from `logs`. + /// + /// Degrades to observer mode (returns `None`) when [`CryptoPlayer::resume`] + /// reports [`MissingPlayerDealing`](DkgError::MissingPlayerDealing): a + /// finalized dealer log carries this node's valid acknowledgement, but the + /// matching private dealing is absent from the secret store (for example, a + /// secret store restored from a backup taken before the ack was emitted). + /// The ceremony has otherwise succeeded, so the node commits the epoch as a + /// share-less verifier rather than crash-looping. Any other error signals + /// genuine corruption and stays fatal. + fn resume_player( + &self, + epoch: Epoch, + signer: C, + info: Info, + logs: &BTreeMap>, + ) -> Option> + where + C: Signer, + M: Faults, + { + match CryptoPlayer::resume::(info, signer, logs, self.dealings(epoch)) { + Ok((player, acks)) => Some(Player { player, acks }), + Err(DkgError::MissingPlayerDealing) => { + warn!( + ?epoch, + "missing private dealing on resume; entering epoch as observer" + ); + None + } + Err(error) => panic!("failed to resume reshare player: {error:?}"), + } + } +} + +/// Appends `event` to the recovery journal for `epoch` and flushes it durably. +async fn append_synced( + events: &mut Journal>, + epoch: Epoch, + event: &Event, +) where + E: BufferPooler + Clock + RuntimeStorage + Metrics, + V: Variant, + P: PublicKey, +{ + let section = epoch.get(); + events + .append(section, event) + .await + .expect("failed to append reshare event"); + events + .sync(section) + .await + .expect("failed to sync reshare event"); +} + +/// Dealer state for one epoch. +pub struct Dealer { + dealer: Option>, + public: DealerPubMsg, + unsent: BTreeMap, + finalized: Option>, +} + +impl Dealer { + /// Records a player ack. + /// + /// Returns [`Verdict::Fault`] if the player signed an invalid ack so the + /// caller can penalize them. A duplicate or unsolicited ack, or one for a + /// round we are not dealing in, is a benign [`Verdict::Skip`]. + pub async fn handle( + &mut self, + store: &mut Store, + epoch: Epoch, + player: C::PublicKey, + ack: PlayerAck, + ) -> Verdict<()> + where + E: BufferPooler + Clock + RuntimeStorage + Metrics, + SS: SecretStore, + { + if !self.unsent.contains_key(&player) { + return Verdict::Skip; + } + let Some(dealer) = &mut self.dealer else { + return Verdict::Skip; + }; + match dealer.receive_player_ack(player.clone(), ack.clone()) { + Ok(()) => {} + Err(DkgError::InvalidAck) => return Verdict::Fault, + Err(_) => return Verdict::Skip, + } + self.unsent.remove(&player); + if store.append_ack(epoch, player, ack).await { + Verdict::Valid(()) + } else { + Verdict::Skip + } + } + + /// Finalizes once and returns true if a new log became available. + pub fn finalize(&mut self) -> bool { + if self.finalized.is_some() { + return false; + } + let Some(dealer) = self.dealer.take() else { + return false; + }; + self.finalized = Some(dealer.finalize::()); + true + } + + /// Returns a cloned finalized log. + pub fn finalized(&self) -> Option> { + self.finalized.clone() + } + + /// Clears a finalized log after it is observed in a finalized block. + pub fn clear_finalized(&mut self) { + self.finalized = None; + } + + /// Returns private dealings that still need to be sent. + pub fn shares_to_distribute( + &self, + ) -> impl Iterator, DealerPrivMsg)> + '_ { + self.unsent + .iter() + .map(|(player, private)| (player.clone(), self.public.clone(), private.clone())) + } +} + +/// Player state for one epoch. +pub struct Player { + player: CryptoPlayer, + acks: BTreeMap>, +} + +impl Player { + /// Handles a dealer message, persisting it before returning the ack. + /// + /// Returns [`Verdict::Fault`] if the dealing is provably invalid so the + /// caller can penalize the dealer. A duplicate dealing is a benign + /// [`Verdict::Skip`]. + pub async fn handle( + &mut self, + store: &mut Store, + epoch: Epoch, + dealer: C::PublicKey, + public: DealerPubMsg, + private: DealerPrivMsg, + ) -> Verdict> + where + E: BufferPooler + Clock + RuntimeStorage + Metrics, + SS: SecretStore, + { + if let Some(ack) = self.acks.get(&dealer) { + return Verdict::Valid(ack.clone()); + } + let ack = match self.player.dealer_message::( + dealer.clone(), + public.clone(), + private.clone(), + ) { + Verdict::Valid(ack) => ack, + Verdict::Skip => return Verdict::Skip, + Verdict::Fault => return Verdict::Fault, + }; + store + .append_dealing(epoch, dealer.clone(), public, private) + .await; + self.acks.insert(dealer, ack.clone()); + Verdict::Valid(ack) + } + + /// Finalizes and returns the public output plus local share. + pub fn finalize( + self, + rng: &mut impl CryptoRng, + logs: Logs, + strategy: &impl Strategy, + ) -> Result<(Output, group::Share), DkgError> + where + M: Faults, + B: BatchVerifier, + { + self.player.finalize::(rng, logs, strategy) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dkg::{ + tests::mocks::MemorySecretStore, + types::{EpochInfo, EpochOutcome}, + }; + use commonware_codec::FixedSize; + use commonware_consensus::types::Epoch; + use commonware_cryptography::{ + Signer, + bls12381::{ + dkg::feldman_desmedt::{Info, Output, deal}, + primitives::{sharing::Mode, variant::MinPk}, + }, + ed25519::{PrivateKey, PublicKey}, + }; + use commonware_runtime::{Runner, Supervisor as _, deterministic}; + use commonware_utils::{N3f1, NZU32, TestRng, ordered::Set}; + + type TestStore = Store; + + fn summary(seed: u8) -> Summary { + let bytes = [seed; Summary::SIZE]; + Summary::read(&mut bytes.as_ref()).expect("valid summary") + } + + fn output(seed: u64) -> Output { + let (output, _) = deal::( + TestRng::new(seed), + Mode::NonZeroCounter, + players(&signers()), + ) + .expect("trusted deal"); + output + } + + fn epoch_info(epoch: Epoch, output: Output) -> EpochInfo { + EpochInfo { + outcome: EpochOutcome::Success, + epoch, + output, + players: Set::default(), + next_players: Set::default(), + } + } + + fn signers() -> Vec { + (0..4).map(PrivateKey::from_seed).collect() + } + + fn players(signers: &[PrivateKey]) -> Set { + Set::from_iter_dedup(signers.iter().map(Signer::public_key)) + } + + async fn init_store( + context: E, + partition: &str, + secret_store: MemorySecretStore, + ) -> TestStore + where + E: BufferPooler + Clock + RuntimeStorage + Metrics, + { + Store::init(context, partition, NZU32!(16), secret_store).await + } + + #[test] + fn commit_epoch_seeds_configured_epoch() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let secret_store = MemorySecretStore::default(); + let mut store = + init_store(context.child("store"), "configured-start", secret_store).await; + store + .commit_epoch(epoch_info(Epoch::new(7), output(1)), summary(1), None) + .await; + + let info = store.current().expect("current epoch"); + assert_eq!(info.epoch, Epoch::new(7)); + }); + } + + #[test] + fn replay_restores_dealings_acks_and_logs() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let secret_store = MemorySecretStore::default(); + let mut store = + init_store(context.child("store"), "replay", secret_store.clone()).await; + let signers = signers(); + let players = players(&signers); + let info = Info::new::( + b"_COMMONWARE_GLUE_DKG_RESHARE_STORE_TEST", + 0, + None, + Mode::NonZeroCounter, + players.clone(), + players.clone(), + ) + .expect("valid info"); + store + .commit_epoch(epoch_info(Epoch::zero(), output(1)), summary(1), None) + .await; + + let dealer_pk = signers[0].public_key(); + let player_pk = signers[1].public_key(); + let mut dealer = store + .create_dealer::( + Epoch::zero(), + signers[0].clone(), + info.clone(), + None, + summary(2), + ) + .expect("dealer"); + let mut player = store + .create_player::(Epoch::zero(), signers[1].clone(), info.clone()) + .expect("player"); + let (_, public, private) = dealer + .shares_to_distribute() + .find(|(recipient, _, _)| *recipient == player_pk) + .expect("dealing for player"); + let Verdict::Valid(ack) = player + .handle( + &mut store, + Epoch::zero(), + dealer_pk.clone(), + public, + private, + ) + .await + else { + panic!("ack"); + }; + assert!(matches!( + dealer + .handle(&mut store, Epoch::zero(), player_pk.clone(), ack) + .await, + Verdict::Valid(()) + )); + assert!(dealer.finalize::()); + let signed = dealer.finalized().expect("signed log"); + let (dealer, log) = signed.check(&info).expect("valid log"); + store.append_log(Epoch::zero(), dealer, log).await; + drop(store); + + let store = init_store(context.child("restart"), "replay", secret_store).await; + // The current epoch is not persisted; the setup state re-derives it from + // finalized blocks, so a restarted store has no current epoch on its own. + assert!(store.current().is_none()); + // The public recovery journal is replayed. + assert_eq!(store.dealings(Epoch::zero()).len(), 1); + assert_eq!(store.acks(Epoch::zero()).len(), 1); + assert_eq!(store.logs(Epoch::zero()).len(), 1); + let replayed_player = store + .create_player::(Epoch::zero(), signers[1].clone(), info) + .expect("player"); + assert_eq!(replayed_player.acks.len(), 1); + }); + } + + #[test] + fn resume_with_missing_dealing_degrades_to_observer() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let secret_store = MemorySecretStore::default(); + let mut store = init_store( + context.child("store"), + "missing-dealing", + secret_store.clone(), + ) + .await; + let signers = signers(); + let players = players(&signers); + let info = Info::new::( + b"_COMMONWARE_GLUE_DKG_RESHARE_STORE_TEST", + 0, + None, + Mode::NonZeroCounter, + players.clone(), + players.clone(), + ) + .expect("valid info"); + store + .commit_epoch(epoch_info(Epoch::zero(), output(1)), summary(1), None) + .await; + + let dealer_pk = signers[0].public_key(); + let mut dealer = store + .create_dealer::( + Epoch::zero(), + signers[0].clone(), + info.clone(), + None, + summary(2), + ) + .expect("dealer"); + + // Collect enough acknowledgements (quorum) that the dealer log records + // an Ack for each acking player rather than degrading to reveals. With + // n=4, f=1 the log tolerates at most one reveal, so three players ack. + for idx in [1usize, 2, 3] { + let player_pk = signers[idx].public_key(); + let mut player = store + .create_player::( + Epoch::zero(), + signers[idx].clone(), + info.clone(), + ) + .expect("player"); + let (_, public, private) = dealer + .shares_to_distribute() + .find(|(recipient, _, _)| *recipient == player_pk) + .expect("dealing for player"); + let Verdict::Valid(ack) = player + .handle( + &mut store, + Epoch::zero(), + dealer_pk.clone(), + public, + private, + ) + .await + else { + panic!("ack"); + }; + assert!(matches!( + dealer + .handle(&mut store, Epoch::zero(), player_pk, ack) + .await, + Verdict::Valid(()) + )); + } + assert!(dealer.finalize::()); + let signed = dealer.finalized().expect("signed log"); + let (dealer, log) = signed.check(&info).expect("valid log"); + store.append_log(Epoch::zero(), dealer, log).await; + drop(store); + + // Restart with an empty secret store: the finalized dealer log (which + // records this player's acknowledgement) replays from public storage, + // but the matching private dealing is gone. Resuming as that player + // must degrade to an observer rather than panic. + let empty_secret_store = MemorySecretStore::default(); + let restarted = init_store( + context.child("restart"), + "missing-dealing", + empty_secret_store, + ) + .await; + assert!(restarted.dealings(Epoch::zero()).is_empty()); + assert_eq!(restarted.logs(Epoch::zero()).len(), 1); + let player = restarted.create_player::( + Epoch::zero(), + signers[1].clone(), + info, + ); + assert!( + player.is_none(), + "a missing private dealing must degrade to observer, not panic" + ); + }); + } + + #[test] + fn protocol_storage_does_not_restore_private_dealings() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let secret_store = MemorySecretStore::default(); + let mut store = init_store( + context.child("store"), + "private-dealing-boundary", + secret_store, + ) + .await; + let signers = signers(); + let players = players(&signers); + let info = Info::new::( + b"_COMMONWARE_GLUE_DKG_RESHARE_STORE_TEST", + 0, + None, + Mode::NonZeroCounter, + players.clone(), + players, + ) + .expect("valid info"); + store + .commit_epoch(epoch_info(Epoch::zero(), output(1)), summary(1), None) + .await; + + let dealer_pk = signers[0].public_key(); + let player_pk = signers[1].public_key(); + let dealer = store + .create_dealer::( + Epoch::zero(), + signers[0].clone(), + info.clone(), + None, + summary(2), + ) + .expect("dealer"); + let mut player = store + .create_player::(Epoch::zero(), signers[1].clone(), info.clone()) + .expect("player"); + let (_, public, private) = dealer + .shares_to_distribute() + .find(|(recipient, _, _)| *recipient == player_pk) + .expect("dealing for player"); + assert!(matches!( + player + .handle(&mut store, Epoch::zero(), dealer_pk, public, private) + .await, + Verdict::Valid(_) + )); + assert_eq!(store.dealings(Epoch::zero()).len(), 1); + drop(store); + + let empty_secret_store = MemorySecretStore::default(); + let restarted = init_store( + context.child("restart"), + "private-dealing-boundary", + empty_secret_store, + ) + .await; + assert!(restarted.dealings(Epoch::zero()).is_empty()); + let replayed_player = restarted + .create_player::(Epoch::zero(), signers[1].clone(), info) + .expect("player"); + assert!(replayed_player.acks.is_empty()); + }); + } + + #[test] + fn protocol_storage_does_not_restore_secret_shares() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let signers = signers(); + let (_, shares) = + deal::(TestRng::new(9), Mode::NonZeroCounter, players(&signers)) + .expect("trusted deal"); + let share = shares + .get_value(&signers[0].public_key()) + .expect("share") + .clone(); + + let secret_store = MemorySecretStore::default(); + let mut store = + init_store(context.child("store"), "secret-boundary", secret_store).await; + store + .commit_epoch( + epoch_info(Epoch::zero(), output(1)), + summary(1), + Some(share), + ) + .await; + assert!(store.share(Epoch::zero()).await.is_some()); + drop(store); + + let empty_secret_store = MemorySecretStore::default(); + let mut restarted = init_store( + context.child("restart"), + "secret-boundary", + empty_secret_store, + ) + .await; + // The current epoch is never persisted; setup re-derives it from + // finalized boundary blocks. The share lived only in the secret store, + // which is now empty, so neither a current epoch nor the share survives. + assert!(restarted.current().is_none()); + assert!(restarted.share(Epoch::zero()).await.is_none()); + }); + } + + #[test] + fn prune_removes_old_protocol_state_and_secret_shares() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let secret_store = MemorySecretStore::default(); + let mut store = init_store(context.child("store"), "prune", secret_store.clone()).await; + let signers = signers(); + let (next_output, shares) = + deal::(TestRng::new(10), Mode::NonZeroCounter, players(&signers)) + .expect("trusted deal"); + let share = shares + .get_value(&signers[0].public_key()) + .expect("share") + .clone(); + + store + .commit_epoch(epoch_info(Epoch::zero(), output(1)), summary(1), None) + .await; + store + .commit_epoch( + epoch_info(Epoch::new(1), next_output), + summary(2), + Some(share), + ) + .await; + + store.prune(Epoch::new(1)).await; + drop(store); + + let store = init_store(context.child("restart"), "prune", secret_store.clone()).await; + // The current epoch is not persisted (the setup state re-derives it), and + // the pruned epoch's public journal and secret material stay pruned across + // the restart. + assert!(store.current().is_none()); + assert!(!store.epochs.contains_key(&Epoch::zero())); + assert_eq!(secret_store.prunes(), vec![Epoch::new(1)]); + assert!(!secret_store.has_share(Epoch::zero())); + }); + } +} diff --git a/glue/src/dkg/tests/dkg/harness.rs b/glue/src/dkg/tests/dkg/harness.rs new file mode 100644 index 00000000000..7e0ec25ae5d --- /dev/null +++ b/glue/src/dkg/tests/dkg/harness.rs @@ -0,0 +1,421 @@ +use super::properties::{DkgOutcome, ExpectedOutcome}; +use crate::{ + dkg::{ + bootstrap, + tests::mocks::{FilteredReceiver, MemorySecretStore}, + types::EpochInfo, + }, + simulate::{ + action::Crash, + engine::{EngineDefinition, InitContext}, + exit::{ExitCondition as _, ProcessedHeightAtLeast}, + plan::PlanBuilder, + processed::ProcessedHeight, + tracker::ProgressTracker, + }, +}; +use commonware_consensus::types::Epoch; +use commonware_cryptography::{ + Signer as _, + bls12381::primitives::{ + group::{Private, Share}, + sharing::Mode, + variant::MinPk, + }, + ed25519, +}; +use commonware_macros::select; +use commonware_math::algebra::Random; +use commonware_p2p::{ + Manager as _, + simulated::{self, Link, Network}, +}; +use commonware_parallel::Sequential; +use commonware_runtime::{ + Clock as _, Handle, Quota, Runner as _, Spawner as _, Supervisor as _, deterministic, + telemetry::metrics::count_running_tasks, +}; +use commonware_utils::{ + NZU32, NZU64, NZUsize, Participant, channel::oneshot, ordered::Set, sync::Mutex, test_rng, +}; +use futures::future::pending; +use std::{ + collections::{BTreeMap, HashSet}, + num::NonZeroU64, + sync::Arc, + time::Duration, +}; + +const NAMESPACE: &[u8] = b"_COMMONWARE_GLUE_DKG_INITIAL_E2E"; +const EPOCH_LENGTH: NonZeroU64 = NZU64!(16); +const TEST_QUOTA: Quota = Quota::per_second(NZU32!(1_000_000)); + +const VOTES: u64 = 0; +const CERTIFICATES: u64 = 1; +const RESOLVER: u64 = 2; +const BACKFILL: u64 = 3; +const BROADCAST: u64 = 4; +const DKG: u64 = 5; + +#[derive(Default)] +struct NodeStateInner { + completed: bool, + info: Option>, +} + +#[derive(Clone)] +pub(super) struct NodeState { + store: MemorySecretStore, + inner: Arc>, +} + +impl NodeState { + pub(super) fn completed(&self) -> bool { + self.inner.lock().completed + } + + pub(super) fn info(&self) -> Option> { + self.inner.lock().info.clone() + } + + pub(super) fn has_share(&self, epoch: Epoch) -> bool { + self.store.has_share(epoch) + } +} + +impl ProcessedHeight for NodeState { + async fn processed_height(&self) -> u64 { + self.inner.lock().completed as u64 + } +} + +pub(super) struct StartedNode { + context: deterministic::Context, + handle: Handle<()>, + completion: oneshot::Receiver>, + state: NodeState, +} + +#[derive(Clone)] +pub(super) struct DkgEngine { + signers: Vec, + filtered_dkg: Arc>, + stores: Arc>>, +} + +impl DkgEngine { + pub(super) fn new(total: u64) -> Self { + let mut signers = (0..total) + .map(ed25519::PrivateKey::from_seed) + .collect::>(); + signers.sort_by_key(|signer| signer.public_key()); + Self { + signers, + filtered_dkg: Arc::default(), + stores: Arc::default(), + } + } + + pub(super) fn with_filtered_dkg(mut self) -> Self { + self.filtered_dkg = Arc::new( + self.signers + .iter() + .map(|signer| signer.public_key()) + .collect(), + ); + self + } + + pub(super) fn participant(&self, index: usize) -> ed25519::PublicKey { + self.signers[index].public_key() + } + + fn signer(&self, public_key: &ed25519::PublicKey) -> ed25519::PrivateKey { + self.signers + .iter() + .find(|signer| signer.public_key() == *public_key) + .expect("participant signer exists") + .clone() + } + + fn participants_set(&self) -> Set { + Set::from_iter_dedup(self.signers.iter().map(|signer| signer.public_key())) + } + + fn store(&self, public_key: &ed25519::PublicKey) -> MemorySecretStore { + self.stores + .lock() + .entry(public_key.clone()) + .or_default() + .clone() + } +} + +impl EngineDefinition for DkgEngine { + type PublicKey = ed25519::PublicKey; + type Engine = StartedNode; + type State = NodeState; + + fn participants(&self) -> Vec { + self.signers + .iter() + .map(|signer| signer.public_key()) + .collect() + } + + fn channels(&self) -> Vec<(u64, Quota)> { + vec![ + (VOTES, TEST_QUOTA), + (CERTIFICATES, TEST_QUOTA), + (RESOLVER, TEST_QUOTA), + (BACKFILL, TEST_QUOTA), + (BROADCAST, TEST_QUOTA), + (DKG, TEST_QUOTA), + ] + } + + async fn init(&self, ctx: InitContext<'_, Self::PublicKey>) -> (Self::Engine, Self::State) { + let InitContext { + context, + index, + public_key, + oracle, + mut channels, + .. + } = ctx; + assert_eq!(channels.len(), 6); + + let store = self.store(public_key); + let state = NodeState { + store: store.clone(), + inner: Arc::default(), + }; + let engine = bootstrap::Engine::<_, MinPk, _, _, _, _>::new( + context.child("dkg"), + bootstrap::Config { + signer: self.signer(public_key), + manager: oracle.manager(), + blocker: oracle.control(public_key.clone()), + secret_store: store, + strategy: Sequential, + namespace: NAMESPACE, + sharing_mode: Mode::NonZeroCounter, + partition_prefix: format!("dkg-{index}"), + participants: self.participants_set(), + blocks_per_epoch: EPOCH_LENGTH, + }, + ); + let (handle, completion) = engine.start( + channels.remove(0), + channels.remove(0), + channels.remove(0), + channels.remove(0), + channels.remove(0), + { + let (sender, receiver) = channels.remove(0); + ( + sender, + if self.filtered_dkg.contains(public_key) { + FilteredReceiver::drop_all(receiver) + } else { + FilteredReceiver::pass(receiver) + }, + ) + }, + ); + + ( + StartedNode { + context, + handle, + completion, + state: state.clone(), + }, + state, + ) + } + + fn start(engine: Self::Engine) -> Handle<()> { + let StartedNode { + context, + handle, + completion, + state, + } = engine; + context.spawn(move |_| async move { + let mut handle = AbortOnDrop(Some(handle)); + select! { + completion = completion => { + let completion = completion.expect("completion channel closed"); + { + let mut inner = state.inner.lock(); + inner.completed = true; + inner.info = completion.info; + } + pending::<()>().await; + }, + result = &mut handle.0.as_mut().expect("handle present") => { + result.expect("DKG engine stopped"); + }, + } + }) + } +} + +struct AbortOnDrop(Option>); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + if let Some(handle) = self.0.take() { + handle.abort(); + } + } +} + +pub(super) fn run_plan( + engine: DkgEngine, + link: Link, + crashes: Vec>, + expected: ExpectedOutcome, +) { + let participants = engine.participants(); + let property = DkgOutcome::new(participants, expected); + let mut builder = PlanBuilder::new(engine) + .link(link) + .required_finalizations(0) + .exit_condition(ProcessedHeightAtLeast::new(1)) + .property(property) + .timeout(Duration::from_secs(60)); + for crash in crashes { + builder = builder.crash(crash); + } + builder.run().expect("DKG simulation"); +} + +pub(super) fn run_restart_completion_state_is_fresh() { + let engine = DkgEngine::new(1); + let public_key = engine.participant(0); + let old_state = NodeState { + store: engine.store(&public_key), + inner: Arc::default(), + }; + let replacement_state = NodeState { + store: engine.store(&public_key), + inner: Arc::default(), + }; + + let share = Share::new(Participant::new(0), Private::random(test_rng())); + old_state.store.seed_share(Epoch::zero(), share); + assert!( + replacement_state.has_share(Epoch::zero()), + "restart must retain the persistent secret store" + ); + + old_state.inner.lock().completed = true; + assert!( + !replacement_state.completed(), + "stale completion changed replacement state" + ); + + let runner = deterministic::Runner::timed(Duration::from_secs(1)); + runner.start(|_| async move { + let tracker = ProgressTracker::::default(); + let states = [&replacement_state]; + let reached = ProcessedHeightAtLeast::new(1) + .reached(&tracker, &states, 1) + .await + .expect("exit condition should evaluate"); + assert!( + !reached, + "exit condition should wait for the current incarnation" + ); + }); +} + +pub(super) fn good_link() -> Link { + Link { + latency: Duration::from_millis(20), + jitter: Duration::from_millis(5), + success_rate: 1.0, + } +} + +pub(super) fn run_closed_network_receiver() { + let runner = deterministic::Runner::timed(Duration::from_secs(5)); + runner.start(|context| async move { + let (network, oracle) = Network::<_, ed25519::PublicKey>::new( + context.child("network"), + simulated::Config { + max_size: 1024 * 1024, + disconnect_on_block: true, + tracked_peer_sets: NZUsize!(1), + }, + ); + network.start(); + + let engine = DkgEngine::new(1); + let public_key = engine.participant(0); + oracle + .manager() + .track(0, Set::from_iter_dedup(engine.participants())); + + let control = oracle.control(public_key.clone()); + let mut channels = Vec::new(); + for (channel, quota) in engine.channels() { + channels.push( + control + .register(channel, quota) + .await + .expect("channel registration failed"), + ); + } + + let _replacement_broadcast = control + .register(BROADCAST, TEST_QUOTA) + .await + .expect("replacement channel registration failed"); + + let store = engine.store(&public_key); + let bootstrap = bootstrap::Engine::<_, MinPk, _, _, _, _>::new( + context.child("dkg"), + bootstrap::Config { + signer: engine.signer(&public_key), + manager: oracle.manager(), + blocker: oracle.control(public_key), + secret_store: store, + strategy: Sequential, + namespace: NAMESPACE, + sharing_mode: Mode::NonZeroCounter, + partition_prefix: "dkg-closed-receiver".into(), + participants: engine.participants_set(), + blocks_per_epoch: EPOCH_LENGTH, + }, + ); + let (mut handle, completion) = bootstrap.start( + channels.remove(0), + channels.remove(0), + channels.remove(0), + channels.remove(0), + channels.remove(0), + channels.remove(0), + ); + + select! { + result = &mut handle => result.expect("bootstrap should stop cleanly"), + _ = context.sleep(Duration::from_secs(1)) => { + panic!("bootstrap did not stop after a supplied receiver closed"); + }, + } + + assert!( + completion.await.is_err(), + "closed receiver should not produce DKG completion" + ); + context.sleep(Duration::from_millis(10)).await; + assert_eq!( + count_running_tasks(&context, "dkg"), + 0, + "bootstrap child actors should be canceled" + ); + }); +} diff --git a/glue/src/dkg/tests/dkg/mod.rs b/glue/src/dkg/tests/dkg/mod.rs new file mode 100644 index 00000000000..e355bd6b41e --- /dev/null +++ b/glue/src/dkg/tests/dkg/mod.rs @@ -0,0 +1,90 @@ +mod harness; +mod properties; + +use crate::simulate::action::{Action, Crash, Schedule}; +use commonware_macros::{test_group, test_traced}; +use commonware_p2p::simulated::Link; +use harness::{ + DkgEngine, good_link, run_closed_network_receiver, run_plan, + run_restart_completion_state_is_fresh, +}; +use properties::ExpectedOutcome; +use std::time::Duration; + +#[test] +fn dkg_e2e_completes_for_all_participants() { + run_plan( + DkgEngine::new(4), + good_link(), + vec![], + ExpectedOutcome::Success, + ); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn dkg_e2e_lossy_network() { + run_plan( + DkgEngine::new(4), + Link { + latency: Duration::from_millis(60), + jitter: Duration::from_millis(20), + success_rate: 0.75, + }, + vec![], + ExpectedOutcome::Success, + ); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn dkg_e2e_filtered_dkg_channel_fails() { + run_plan( + DkgEngine::new(4).with_filtered_dkg(), + good_link(), + vec![], + ExpectedOutcome::Failure, + ); +} + +#[test] +fn dkg_e2e_closed_network_receiver_stops_engine() { + run_closed_network_receiver(); +} + +#[test] +fn dkg_e2e_restart_completion_state_is_fresh() { + run_restart_completion_state_is_fresh(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn dkg_e2e_scheduled_restart() { + let engine = DkgEngine::new(4); + let restarted = engine.participant(0); + run_plan( + engine, + good_link(), + vec![Crash::Schedule( + Schedule::new() + .at(Duration::from_millis(80), Action::Crash(restarted.clone())) + .at(Duration::from_millis(250), Action::Restart(restarted)), + )], + ExpectedOutcome::Success, + ); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn dkg_e2e_random_crashes() { + run_plan( + DkgEngine::new(4), + good_link(), + vec![Crash::Random { + frequency: Duration::from_millis(250), + downtime: Duration::from_millis(50), + count: 1, + }], + ExpectedOutcome::Success, + ); +} diff --git a/glue/src/dkg/tests/dkg/properties.rs b/glue/src/dkg/tests/dkg/properties.rs new file mode 100644 index 00000000000..c6c8a47db05 --- /dev/null +++ b/glue/src/dkg/tests/dkg/properties.rs @@ -0,0 +1,109 @@ +use super::harness::NodeState; +use crate::{ + dkg::types::EpochOutcome, + simulate::{property::Property, tracker::ProgressTracker}, +}; +use commonware_consensus::types::Epoch; +use commonware_cryptography::{ + bls12381::{dkg::feldman_desmedt::Output, primitives::variant::MinPk}, + ed25519, +}; +use commonware_utils::ordered::Set; +use std::{future::Future, pin::Pin}; + +#[derive(Clone, Copy)] +pub(super) enum ExpectedOutcome { + Success, + Failure, +} + +#[derive(Clone)] +pub(super) struct DkgOutcome { + participants: Set, + expected: ExpectedOutcome, +} + +impl DkgOutcome { + pub(super) fn new(participants: Vec, expected: ExpectedOutcome) -> Self { + Self { + participants: Set::from_iter_dedup(participants), + expected, + } + } +} + +impl Property for DkgOutcome { + fn name(&self) -> &str { + match self.expected { + ExpectedOutcome::Success => "dkg_success", + ExpectedOutcome::Failure => "dkg_failure", + } + } + + fn check<'a>( + &'a self, + _tracker: &'a ProgressTracker, + states: &'a [&'a NodeState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + if states.len() != self.participants.len() { + return Err(format!( + "expected {} active states, got {}", + self.participants.len(), + states.len() + )); + } + + let mut expected: Option> = None; + for state in states { + if !state.completed() { + return Err("state did not complete".into()); + } + + let Some(info) = state.info() else { + if matches!(self.expected, ExpectedOutcome::Failure) { + if state.has_share(Epoch::zero()) { + return Err("failed DKG persisted epoch zero share".into()); + } + continue; + } + return Err("missing DKG info".into()); + }; + + if matches!(self.expected, ExpectedOutcome::Failure) { + return Err("failed DKG produced epoch info".into()); + } + if info.outcome != EpochOutcome::Success { + return Err(format!("unexpected outcome: {:?}", info.outcome)); + } + if info.epoch != Epoch::zero() { + return Err(format!("unexpected epoch: {:?}", info.epoch)); + } + if info.output.players() != &self.participants { + return Err("output players did not match participants".into()); + } + if info.players != self.participants { + return Err("epoch info players did not match participants".into()); + } + if !info.next_players.is_empty() { + return Err("DKG next_players should be empty".into()); + } + if !info.output.revealed().is_empty() { + return Err("DKG revealed shares".into()); + } + if let Some(expected) = &expected { + if &info.output != expected { + return Err("participants produced different outputs".into()); + } + } else { + expected = Some(info.output); + } + + if !state.has_share(Epoch::zero()) { + return Err("secret store missing epoch zero share".into()); + } + } + Ok(()) + }) + } +} diff --git a/glue/src/dkg/tests/mocks.rs b/glue/src/dkg/tests/mocks.rs new file mode 100644 index 00000000000..6c8ad4f71bf --- /dev/null +++ b/glue/src/dkg/tests/mocks.rs @@ -0,0 +1,584 @@ +#![allow(dead_code)] + +use crate::dkg::{ + Registrar, ReshareBlock, SecretStore, orchestrator, + types::{Payload, SchemeInfo}, +}; +use bytes::{Buf, BufMut}; +use commonware_actor::Feedback; +use commonware_codec::{ + Codec, Decode, Encode, EncodeSize, Error as CodecError, Read, ReadExt, Write, varint::UInt, +}; +use commonware_consensus::{ + Automaton, Block, CertifiableAutomaton, Heightable, Relay, Reporter, + marshal::{Update, core::Mailbox as MarshalMailbox, standard::Standard}, + simplex::{self, Plan, elector::RoundRobin, mocks::scheme, types::Context}, + types::{Epoch, Height, Round, View, ViewDelta}, +}; +use commonware_cryptography::{ + Digest, Digestible, Hasher, PublicKey as CryptoPublicKey, Signer, + bls12381::{ + dkg::feldman_desmedt::DealerPrivMsg, + primitives::{ + group::Share, + variant::{MinPk, Variant}, + }, + }, + certificate::ConstantProvider, + ed25519::{PrivateKey, PublicKey}, + sha256::{Digest as Sha256Digest, Sha256}, + transcript::Summary, +}; +use commonware_p2p::{ + Message as P2pMessage, Receiver, + simulated::{Control, Manager as SimManager}, + utils::mux, +}; +use commonware_parallel::Sequential; +use commonware_runtime::deterministic; +use commonware_utils::{ + Acknowledgement, NZU16, NZUsize, + channel::{fallible::OneshotExt, oneshot}, + sync::Mutex, +}; +use std::{ + collections::{BTreeMap, HashSet}, + num::NonZeroU32, + sync::Arc, + time::Duration, +}; + +pub(crate) type TestDigest = Sha256Digest; +pub(crate) type TestPublicKey = PublicKey; +pub(crate) type TestSigner = PrivateKey; +pub(crate) type TestContext = Context; +pub(crate) type TestBlock = MockBlock; +pub(crate) type TestMarshalVariant = Standard; +pub(crate) type TestBlsVariant = MinPk; +pub(crate) type TestScheme = scheme::Scheme; +pub(crate) type TestProvider = ConstantProvider; +pub(crate) type TestElector = RoundRobin; +pub(crate) type TestStrategy = Sequential; +pub(crate) type TestBlocker = Control; +pub(crate) type TestManager = SimManager; +pub(crate) type TestMailbox = orchestrator::Mailbox; +pub(crate) type TestMarshalMailbox = MarshalMailbox; +pub(crate) type TestActor = orchestrator::Actor< + deterministic::Context, + TestBlocker, + TestManager, + TestProvider, + TestMarshalVariant, + TestBlsVariant, + TestSigner, + MockApplication, + TestElector, + TestStrategy, +>; + +const NAMESPACE: &[u8] = b"_COMMONWARE_GLUE_DKG_ORCHESTRATOR_TEST"; + +#[derive(Debug)] +pub(crate) struct FilteredReceiver { + inner: R, + filter: Filter, +} + +#[derive(Debug)] +enum Filter { + None, + All, + Epochs(Arc>), +} + +impl FilteredReceiver { + pub(crate) const fn pass(inner: R) -> Self { + Self { + inner, + filter: Filter::None, + } + } + + pub(crate) const fn drop_all(inner: R) -> Self { + Self { + inner, + filter: Filter::All, + } + } + + pub(crate) const fn epochs(inner: R, epochs: Arc>) -> Self { + Self { + inner, + filter: Filter::Epochs(epochs), + } + } +} + +impl Receiver for FilteredReceiver { + type Error = R::Error; + type PublicKey = R::PublicKey; + + async fn recv(&mut self) -> Result, Self::Error> { + loop { + let message = self.inner.recv().await?; + match &self.filter { + Filter::None => return Ok(message), + Filter::All => {} + Filter::Epochs(epochs) => { + let (_, bytes) = &message; + let (epoch, _) = + mux::parse(bytes.clone()).expect("failed to parse mux message"); + if !epochs.contains(&epoch) { + return Ok(message); + } + } + } + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub(crate) struct MockBlock { + context: C, + parent: D, + height: Height, + timestamp: u64, + payload: Option, + digest: D, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub(crate) struct EncodedPayload { + max_participants: NonZeroU32, + bytes: Vec, +} + +impl EncodedPayload { + pub(crate) fn new( + max_participants: NonZeroU32, + payload: Payload, + ) -> Self { + Self { + max_participants, + bytes: payload.encode().to_vec(), + } + } + + fn decode(&self) -> Option> { + Payload::decode_cfg(self.bytes.as_slice(), &self.max_participants).ok() + } + + fn write(&self, writer: &mut impl BufMut) { + UInt(self.max_participants.get()).write(writer); + UInt(u32::try_from(self.bytes.len()).expect("payload too large")).write(writer); + writer.put_slice(&self.bytes); + } + + fn read(reader: &mut impl Buf) -> Result { + let max_participants = NonZeroU32::new(UInt::::read(reader)?.into()).ok_or( + CodecError::Invalid("EncodedPayload", "max participants must be non-zero"), + )?; + let len: u32 = UInt::read(reader)?.into(); + let len = len as usize; + if reader.remaining() < len { + return Err(CodecError::EndOfBuffer); + } + let bytes = reader.copy_to_bytes(len).to_vec(); + Ok(Self { + max_participants, + bytes, + }) + } + + fn encode_size(&self) -> usize { + UInt(self.max_participants.get()).encode_size() + + UInt(u32::try_from(self.bytes.len()).expect("payload too large")).encode_size() + + self.bytes.len() + } +} + +impl MockBlock { + pub(crate) fn new>( + context: C, + parent: D, + height: Height, + timestamp: u64, + ) -> Self { + Self::from_parts::(context, parent, height, timestamp, None) + } + + pub(crate) fn with_payload( + self, + max_participants: NonZeroU32, + payload: Payload, + ) -> Self + where + H: Hasher, + V: Variant, + S: Signer, + { + Self::from_parts::( + self.context, + self.parent, + self.height, + self.timestamp, + Some(EncodedPayload::new(max_participants, payload)), + ) + } + + pub(crate) const fn context(&self) -> &C { + &self.context + } + + fn from_parts>( + context: C, + parent: D, + height: Height, + timestamp: u64, + payload: Option, + ) -> Self { + let mut hasher = H::new(); + hasher.update(&parent); + hasher.update(&height.get().to_be_bytes()); + hasher.update(&context.encode()); + hasher.update(×tamp.to_be_bytes()); + match &payload { + Some(payload) => { + hasher.update(&[1]); + hasher.update(&payload.max_participants.get().to_be_bytes()); + hasher.update( + &u32::try_from(payload.bytes.len()) + .expect("payload too large") + .to_be_bytes(), + ); + hasher.update(&payload.bytes); + } + None => { + hasher.update(&[0]); + } + } + let digest = hasher.finalize(); + Self { + context, + parent, + height, + timestamp, + payload, + digest, + } + } +} + +impl Write for MockBlock { + fn write(&self, writer: &mut impl BufMut) { + self.context.write(writer); + self.parent.write(writer); + self.height.write(writer); + UInt(self.timestamp).write(writer); + self.payload.is_some().write(writer); + if let Some(log) = &self.payload { + log.write(writer); + } + self.digest.write(writer); + } +} + +impl> Read for MockBlock { + type Cfg = (); + + fn read_cfg(reader: &mut impl Buf, _: &Self::Cfg) -> Result { + Ok(Self { + context: C::read(reader)?, + parent: D::read(reader)?, + height: Height::read(reader)?, + timestamp: UInt::read(reader)?.into(), + payload: if bool::read(reader)? { + Some(EncodedPayload::read(reader)?) + } else { + None + }, + digest: D::read(reader)?, + }) + } +} + +impl EncodeSize for MockBlock { + fn encode_size(&self) -> usize { + self.context.encode_size() + + self.parent.encode_size() + + self.height.encode_size() + + UInt(self.timestamp).encode_size() + + self.payload.is_some().encode_size() + + self.payload.as_ref().map_or(0, EncodedPayload::encode_size) + + self.digest.encode_size() + } +} + +impl Digestible for MockBlock { + type Digest = D; + + fn digest(&self) -> D { + self.digest + } +} + +impl Heightable for MockBlock { + fn height(&self) -> Height { + self.height + } +} + +impl + Clone + Send + Sync + 'static> Block for MockBlock { + fn parent(&self) -> Self::Digest { + self.parent + } +} + +impl ReshareBlock for MockBlock +where + D: Digest, + C: Codec + Clone + Send + Sync + 'static, +{ + type Variant = TestBlsVariant; + type Signer = TestSigner; + + fn payload(&self) -> Option> { + self.payload.as_ref()?.decode() + } +} + +#[derive(Clone, Default)] +pub(crate) struct MockApplication { + broadcasts: Arc>>, + proposals: Arc>>, +} + +impl MockApplication { + pub(crate) fn broadcasts(&self) -> Vec { + self.broadcasts.lock().clone() + } + + pub(crate) fn proposals(&self) -> Vec { + self.proposals.lock().clone() + } +} + +impl Automaton for MockApplication { + type Context = TestContext; + type Digest = TestDigest; + + async fn propose(&mut self, _context: Self::Context) -> oneshot::Receiver { + let (sender, receiver) = oneshot::channel(); + self.proposals.lock().push(_context); + sender.send_lossy(Sha256::hash(b"proposal")); + receiver + } + + async fn verify( + &mut self, + _context: Self::Context, + _payload: Self::Digest, + ) -> oneshot::Receiver { + let (sender, receiver) = oneshot::channel(); + sender.send_lossy(true); + receiver + } +} + +impl CertifiableAutomaton for MockApplication {} + +impl Relay for MockApplication { + type Digest = TestDigest; + type PublicKey = TestPublicKey; + type Plan = Plan; + + fn broadcast(&mut self, payload: Self::Digest, _plan: Self::Plan) -> Feedback { + self.broadcasts.lock().push(payload); + Feedback::Ok + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum ConsumerEvent { + Enter(Epoch), + Exit(Epoch), +} + +#[derive(Clone, Default)] +pub(crate) struct MockConsumer { + events: Arc>>, +} + +impl MockConsumer { + pub(crate) fn events(&self) -> Vec { + self.events.lock().clone() + } +} + +impl Registrar for MockConsumer { + type Variant = TestBlsVariant; + type PublicKey = TestPublicKey; + + async fn register(&self, epoch: Epoch, _info: SchemeInfo) { + self.events.lock().push(ConsumerEvent::Enter(epoch)); + } +} + +#[derive(Clone, Default)] +pub(crate) struct MarshalApplication { + blocks: Arc>>>, +} + +impl MarshalApplication { + pub(crate) fn blocks(&self) -> BTreeMap> { + self.blocks.lock().clone() + } +} + +impl Reporter for MarshalApplication { + type Activity = Update; + + fn report(&mut self, activity: Self::Activity) -> Feedback { + if let Update::Block(block, ack) = activity { + self.blocks.lock().insert(block.height(), block); + ack.acknowledge(); + } + Feedback::Ok + } +} + +pub(crate) struct SchemeFixture { + pub(crate) participants: Vec, + pub(crate) schemes: Vec, + pub(crate) provider: TestProvider, +} + +pub(crate) fn scheme_fixture(context: &mut deterministic::Context) -> SchemeFixture { + scheme_fixture_n(context, 1) +} + +pub(crate) fn scheme_fixture_n(context: &mut deterministic::Context, n: u32) -> SchemeFixture { + let fixture = scheme::fixture(context, NAMESPACE, n); + let provider = ConstantProvider::new(fixture.schemes[0].clone()); + SchemeFixture { + participants: fixture.participants, + schemes: fixture.schemes, + provider, + } +} + +pub(crate) fn genesis_block(leader: TestPublicKey) -> TestBlock { + let digest = Sha256::hash(b""); + let context = TestContext { + round: Round::new(Epoch::zero(), View::zero()), + leader, + parent: (View::zero(), digest), + }; + TestBlock::new::(context, digest, Height::zero(), 0) +} + +pub(crate) fn simplex_config() -> orchestrator::SimplexConfig { + orchestrator::SimplexConfig { + elector: TestElector::default(), + mailbox_size: NZUsize!(16), + replay_buffer: NZUsize!(1024), + write_buffer: NZUsize!(1024), + page_cache_page_size: NZU16!(1024), + page_cache_pages: NZUsize!(8), + leader_timeout: Duration::from_millis(100), + certification_timeout: Duration::from_millis(200), + timeout_retry: Duration::from_millis(500), + fetch_timeout: Duration::from_millis(100), + fetch_concurrent: NZUsize!(2), + activity_timeout: ViewDelta::new(8), + skip_timeout: ViewDelta::new(2), + forwarding: simplex::ForwardingPolicy::Disabled, + } +} + +/// In-memory [`SecretStore`] for tests. +/// +/// Dealings are keyed by the encoded dealer key so the store works with any +/// [`PublicKey`](CryptoPublicKey). Pruned epochs are recorded for assertions. +#[derive(Clone, Default)] +pub(crate) struct MemorySecretStore { + inner: Arc>, +} + +#[derive(Default)] +struct MemorySecretStoreInner { + shares: BTreeMap, + seeds: BTreeMap, + dealings: BTreeMap<(Epoch, Vec), DealerPrivMsg>, + prunes: Vec, +} + +impl MemorySecretStore { + /// Returns whether a share is held for `epoch`. + pub(crate) fn has_share(&self, epoch: Epoch) -> bool { + self.inner.lock().shares.contains_key(&epoch) + } + + /// Returns the epochs passed to [`SecretStore::prune`], in call order. + pub(crate) fn prunes(&self) -> Vec { + self.inner.lock().prunes.clone() + } + + /// Pre-seeds a share for `epoch`, for tests that install state before the + /// actor starts. + pub(crate) fn seed_share(&self, epoch: Epoch, share: Share) { + self.inner.lock().shares.insert(epoch, share); + } +} + +impl SecretStore for MemorySecretStore { + async fn put_share(&mut self, epoch: Epoch, share: Share) { + self.inner.lock().shares.insert(epoch, share); + } + + async fn get_share(&mut self, epoch: Epoch) -> Option { + self.inner.lock().shares.get(&epoch).cloned() + } + + async fn put_seed(&mut self, epoch: Epoch, seed: Summary) { + self.inner.lock().seeds.insert(epoch, seed); + } + + async fn get_seed(&mut self, epoch: Epoch) -> Option

{ + self.inner.lock().seeds.get(&epoch).cloned() + } + + async fn put_dealing( + &mut self, + epoch: Epoch, + dealer: P, + private: DealerPrivMsg, + ) { + self.inner + .lock() + .dealings + .insert((epoch, dealer.encode().to_vec()), private); + } + + async fn get_dealing( + &mut self, + epoch: Epoch, + dealer: &P, + ) -> Option { + self.inner + .lock() + .dealings + .get(&(epoch, dealer.encode().to_vec())) + .cloned() + } + + async fn prune(&mut self, min: Epoch) { + let mut inner = self.inner.lock(); + inner.prunes.push(min); + inner.shares.retain(|epoch, _| *epoch >= min); + inner.seeds.retain(|epoch, _| *epoch >= min); + inner.dealings.retain(|(epoch, _), _| *epoch >= min); + } +} diff --git a/glue/src/dkg/tests/mod.rs b/glue/src/dkg/tests/mod.rs new file mode 100644 index 00000000000..611337ac7ab --- /dev/null +++ b/glue/src/dkg/tests/mod.rs @@ -0,0 +1,3 @@ +mod dkg; +pub(crate) mod mocks; +mod reshare; diff --git a/glue/src/dkg/tests/reshare/harness.rs b/glue/src/dkg/tests/reshare/harness.rs new file mode 100644 index 00000000000..c3b66e90b05 --- /dev/null +++ b/glue/src/dkg/tests/reshare/harness.rs @@ -0,0 +1,1105 @@ +use crate::{ + dkg::{ + ParticipantsProvider, Registrar, ReshareBlock, SecretStore, anchor, + fence::Fence, + orchestrator, + reshare::{self, Input as ReshareInput}, + tests::mocks::{FilteredReceiver, MemorySecretStore}, + types::*, + }, + simulate::{ + engine::{EngineDefinition, InitContext}, + processed::ProcessedHeight, + reporter::MonitorReporter, + }, + stateful::{ + Application, Config as StatefulConfig, Input, Proposed, Stateful as StatefulActor, + SyncPlan, + db::{ + DatabaseSet, Merkleized as _, SyncEngineConfig, Unmerkleized as _, + p2p::standard as qmdb_resolver, + }, + probe::{Config as ProbeConfig, Probe}, + }, +}; +use commonware_broadcast::buffered; +use commonware_codec::{Encode, EncodeSize, Error as CodecError, Read, ReadExt as _, Write}; +use commonware_consensus::{ + Block as ConsensusBlock, CertifiableBlock, Heightable, Reporters, + marshal::{ + self, + ancestry::Ancestry, + core::{Actor as MarshalActor, CommitmentFallback, Mailbox as MarshalMailbox}, + resolver::p2p as marshal_resolver, + standard::{Deferred, Standard}, + }, + simplex::{ + self, + config::ForwardingPolicy, + elector::RoundRobin, + types::{Context, Finalization}, + }, + types::{Epoch, Epocher as _, FixedEpocher, Height, Round, View, ViewDelta}, +}; +use commonware_cryptography::{ + Digest as _, Digestible, Hasher, Sha256, Signer as _, + bls12381::{ + dkg::feldman_desmedt::deal, + primitives::{group::Share, sharing::Mode, variant::MinPk}, + }, + certificate::{Provider as CertificateProvider, Scoped}, + ed25519, + sha256::{self, Digest as Sha256Digest}, +}; +use commonware_formatting::hex; +use commonware_math::algebra::Random; +use commonware_p2p::utils::mux::{Builder, Muxer}; +use commonware_parallel::Sequential; +use commonware_runtime::{ + Buf, BufMut, BufferPooler, Clock, Handle, Metrics, Quota, Spawner, Storage, Supervisor as _, + buffer::paged::CacheRef, deterministic::Context as DeterministicContext, +}; +use commonware_storage::{ + archive::prunable, + journal::contiguous::fixed::Config as FixedLogConfig, + mmr::{self, Location, full::Config as MmrJournalConfig}, + qmdb::{ + any::{FixedConfig, unordered::fixed}, + sync::Target, + }, + translator::TwoCap, +}; +use commonware_utils::{ + N3f1, NZDuration, NZU16, NZU32, NZU64, NZUsize, TestRng, non_empty_range, + ordered::{Map, Set}, + range::NonEmptyRange, + sync::{Mutex, TracedAsyncRwLock}, + test_rng, +}; +use rand::Rng; +use std::{ + collections::{BTreeMap, HashMap, HashSet, btree_map::Entry}, + marker::PhantomData, + num::{NonZeroU16, NonZeroU32, NonZeroU64, NonZeroUsize}, + sync::Arc, + time::Duration, +}; + +type Qmdb = + fixed::Db; +type Database = Arc>>; +type Scheme = simplex::scheme::bls12381_threshold::vrf::Scheme; +type MarshalVariant = Standard; +type Marshal = MarshalMailbox; + +pub(super) const EPOCH_LENGTH: NonZeroU64 = NZU64!(32); +const NAMESPACE: &[u8] = b"_COMMONWARE_GLUE_DKG_RESHARE_E2E"; +const PAGE_SIZE: NonZeroU16 = NZU16!(1024); +const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(10); +const IO_BUFFER_SIZE: NonZeroUsize = NZUsize!(2048); +const TEST_QUOTA: Quota = Quota::per_second(NZU32!(1_000_000)); +const MAX_PARTICIPANTS: NonZeroU32 = NZU32!(16); + +const VOTE_CHANNEL: u64 = 0; +const CERTIFICATE_CHANNEL: u64 = 1; +const RESOLVER_CHANNEL: u64 = 2; +const BACKFILL_CHANNEL: u64 = 3; +const BROADCAST_CHANNEL: u64 = 4; +const QMDB_CHANNEL: u64 = 5; +const DKG_CHANNEL: u64 = 6; +const PROBE_CHANNEL: u64 = 7; +const ANCHOR_BOUNDARY_CHANNEL: u64 = 8; + +#[derive(Clone, PartialEq, Eq)] +pub(super) struct Block { + context: Context, + parent: sha256::Digest, + height: Height, + state_root: sha256::Digest, + range: NonEmptyRange, + payload: Option>, +} + +impl Write for Block { + fn write(&self, buf: &mut impl BufMut) { + self.context.write(buf); + self.parent.write(buf); + self.height.write(buf); + self.state_root.write(buf); + self.range.write(buf); + self.payload.write(buf); + } +} + +impl EncodeSize for Block { + fn encode_size(&self) -> usize { + self.context.encode_size() + + self.parent.encode_size() + + self.height.encode_size() + + self.state_root.encode_size() + + self.range.encode_size() + + self.payload.encode_size() + } +} + +impl Read for Block { + type Cfg = (); + + fn read_cfg(buf: &mut impl Buf, _: &Self::Cfg) -> Result { + Ok(Self { + context: Context::read(buf)?, + parent: sha256::Digest::read(buf)?, + height: Height::read(buf)?, + state_root: sha256::Digest::read(buf)?, + range: NonEmptyRange::read(buf)?, + payload: Option::>::read_cfg( + buf, + &MAX_PARTICIPANTS, + )?, + }) + } +} + +impl Digestible for Block { + type Digest = sha256::Digest; + + fn digest(&self) -> sha256::Digest { + Sha256::hash(&self.encode()) + } +} + +impl Heightable for Block { + fn height(&self) -> Height { + self.height + } +} + +impl ConsensusBlock for Block { + fn parent(&self) -> sha256::Digest { + self.parent + } +} + +impl CertifiableBlock for Block { + type Context = Context; + + fn context(&self) -> Self::Context { + self.context.clone() + } +} + +impl ReshareBlock for Block { + type Variant = MinPk; + type Signer = ed25519::PrivateKey; + + fn payload(&self) -> Option> { + self.payload.clone() + } +} + +impl Block { + fn genesis(leader: ed25519::PublicKey, info: EpochInfo) -> Self { + Self { + context: Context { + round: Round::new(Epoch::zero(), View::zero()), + leader, + parent: (View::zero(), sha256::Digest::EMPTY), + }, + parent: sha256::Digest::EMPTY, + height: Height::zero(), + state_root: empty_db_root(), + range: non_empty_range!(Location::new(0), Location::new(1)), + payload: Some(Payload::EpochInfo(info)), + } + } +} + +#[derive(Clone)] +struct App { + genesis: Block, +} + +impl App { + async fn execute( + height: Height, + mut batches: as DatabaseSet>::Unmerkleized, + ) -> as DatabaseSet>::Merkleized { + let key = Sha256::hash(b"height"); + batches = batches.write(key, Some(u64_to_digest(height.get()))); + batches.merkleize().await.unwrap() + } +} + +impl Application for App { + type SigningScheme = Scheme; + type Context = Context; + type Block = Block; + type Databases = Database; + type Provider = (); + type Input = ReshareInput<(), MinPk, ed25519::PrivateKey>; + + async fn genesis(&mut self) -> Self::Block { + self.genesis.clone() + } + + async fn propose( + &mut self, + context: (E, Self::Context), + ancestry: impl Ancestry, + batches: >::Unmerkleized, + input: Input, + ) -> Option> { + let parent = ancestry.peek()?.clone(); + let height = Height::new(parent.height().get() + 1); + // The reshare::Application wrapper selected and fetched the payload. + let payload = input.parent.payload; + let merkleized = Self::execute(height, batches).await; + let bounds = merkleized.bounds(); + let block = Block { + context: context.1, + parent: parent.digest(), + height, + state_root: merkleized.root(), + range: non_empty_range!(bounds.inactivity_floor, Location::new(bounds.total_size)), + payload, + }; + Some(Proposed { block, merkleized }) + } + + async fn verify( + &mut self, + _context: (E, Self::Context), + ancestry: impl Ancestry, + batches: >::Unmerkleized, + ) -> Option<>::Merkleized> { + // Reshare final-block payload validation is enforced by the surrounding + // reshare::Application wrapper; this inner app only executes state. + let tip = ancestry.peek()?.clone(); + let merkleized = Self::execute(tip.height(), batches).await; + Some(merkleized) + } + + async fn apply( + &mut self, + _context: (E, Self::Context), + block: &Self::Block, + batches: >::Unmerkleized, + ) -> >::Merkleized { + Self::execute(block.height(), batches).await + } + + fn sync_targets(block: &Self::Block) -> >::SyncTargets { + Target::new(block.state_root, block.range.clone()) + } +} + +#[derive(Clone)] +struct DynamicProvider { + schemes: Arc>>>, +} + +impl DynamicProvider { + pub(super) fn new() -> Self { + Self { + schemes: Arc::new(Mutex::new(HashMap::new())), + } + } + + fn register(&self, epoch: Epoch, scheme: Scheme) { + self.schemes.lock().insert(epoch, Arc::new(scheme)); + } +} + +impl CertificateProvider for DynamicProvider { + type Scope = Epoch; + type Scheme = Scheme; + + fn scoped(&self, scope: Self::Scope) -> Option> { + self.schemes.lock().get(&scope).cloned().map(Scoped::scheme) + } + + fn scheme(&self, scope: Self::Scope) -> Option> { + self.schemes.lock().get(&scope).cloned() + } +} + +#[derive(Clone)] +struct TestRegistrar { + provider: DynamicProvider, + events: Arc>>>, + public_key: ed25519::PublicKey, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RegistrationRole { + Signer, + Verifier, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Registration { + pub(super) epoch: Epoch, + pub(super) role: RegistrationRole, +} + +impl Registrar for TestRegistrar { + type Variant = MinPk; + type PublicKey = ed25519::PublicKey; + + async fn register(&self, epoch: Epoch, info: SchemeInfo) { + let (scheme, role) = match info { + SchemeInfo::Verifier { + participants, + sharing, + } => ( + Scheme::verifier(NAMESPACE, participants, sharing), + RegistrationRole::Verifier, + ), + SchemeInfo::Signer { + participants, + sharing, + share, + } => ( + Scheme::signer(NAMESPACE, participants, sharing, share) + .expect("share must match participant set"), + RegistrationRole::Signer, + ), + }; + self.provider.register(epoch, scheme); + self.events + .lock() + .entry(self.public_key.clone()) + .or_default() + .push(Registration { epoch, role }); + } +} + +#[derive(Clone)] +struct ScheduleProvider { + pub(super) schedule: Arc, +} + +impl ParticipantsProvider for ScheduleProvider { + type PublicKey = ed25519::PublicKey; + + async fn participants(&mut self, epoch: Epoch) -> Set { + self.schedule.players(epoch) + } +} + +#[derive(Clone)] +pub(super) struct CommitteeSchedule { + participants: Vec, + committee_sizes: Vec, +} + +impl CommitteeSchedule { + pub(super) fn players(&self, epoch: Epoch) -> Set { + let offset = epoch.get() as usize % self.participants.len(); + let committee_size = + self.committee_sizes[epoch.get() as usize % self.committee_sizes.len()]; + let players = (0..committee_size) + .map(|i| self.participants[(offset + i) % self.participants.len()].clone()); + Set::from_iter_dedup(players) + } +} + +#[derive(Clone)] +pub(super) struct ReshareEngine { + signers: Vec, + pub(super) participants: Vec, + pub(super) schedule: Arc, + initial: Arc, + sharing_mode: Mode, + stores: Arc>>, + pub(super) registrations: Arc>>>, + pub(super) state_syncs: Arc>>, + state_sync_floor: Option, + marshals: Arc>>, + failures: Arc>, +} + +pub(super) struct ValidatorEngine { + context: DeterministicContext, + handles: ValidatorHandles, +} + +struct ValidatorHandles { + anchor: Handle<()>, + probe: Handle<()>, + qmdb: Handle<()>, + reshare: Handle<()>, + orchestrator: Handle<()>, + marshal: Handle<()>, + stateful: Handle<()>, +} + +impl ValidatorHandles { + async fn join(mut self) { + futures::try_join!( + &mut self.anchor, + &mut self.probe, + &mut self.qmdb, + &mut self.reshare, + &mut self.orchestrator, + &mut self.marshal, + &mut self.stateful, + ) + .expect("validator actor failed"); + } +} + +impl Drop for ValidatorHandles { + fn drop(&mut self) { + self.anchor.abort(); + self.probe.abort(); + self.qmdb.abort(); + self.reshare.abort(); + self.orchestrator.abort(); + self.marshal.abort(); + self.stateful.abort(); + } +} + +#[derive(Clone)] +struct InitialState { + info: EpochInfo, + shares: Map, +} + +impl InitialState { + async fn register_epoch_zero( + &self, + provider: &DynamicProvider, + store: &MemorySecretStore, + ) -> RegistrationRole { + let mut store = store.clone(); + let participants = self.info.output.players(); + let sharing = self.info.output.public(); + store.get_share(Epoch::zero()).await.map_or_else( + || { + provider.register( + Epoch::zero(), + Scheme::verifier(NAMESPACE, participants.clone(), sharing.clone()), + ); + RegistrationRole::Verifier + }, + |share| { + provider.register( + Epoch::zero(), + Scheme::signer(NAMESPACE, participants.clone(), sharing.clone(), share) + .expect("initial signer share"), + ); + RegistrationRole::Signer + }, + ) + } +} + +impl ReshareEngine { + pub(super) fn new() -> Self { + Self::with_committee(5, 4) + } + + pub(super) fn with_committee(total: u32, committee_size: usize) -> Self { + Self::with_committees(total, vec![committee_size]) + } + + pub(super) fn with_committees(total: u32, committee_sizes: Vec) -> Self { + assert!(!committee_sizes.is_empty()); + for committee_size in &committee_sizes { + assert!(*committee_size > 0); + assert!(*committee_size <= total as usize); + } + let mut rng = test_rng(); + let signers = (0..total) + .map(|_| ed25519::PrivateKey::random(&mut rng)) + .collect::>(); + let participants = signers.iter().map(|s| s.public_key()).collect::>(); + let schedule = Arc::new(CommitteeSchedule { + participants: participants.clone(), + committee_sizes, + }); + let players = schedule.players(Epoch::zero()); + let (output, shares) = + deal::(TestRng::new(10), Mode::NonZeroCounter, players.clone()) + .expect("trusted initial deal"); + let info = EpochInfo { + outcome: EpochOutcome::Success, + epoch: Epoch::zero(), + output, + players, + next_players: schedule.players(Epoch::new(1)), + }; + Self { + signers, + participants, + schedule, + initial: Arc::new(InitialState { info, shares }), + sharing_mode: Mode::NonZeroCounter, + stores: Arc::new(Mutex::new(BTreeMap::new())), + registrations: Arc::new(Mutex::new(BTreeMap::new())), + state_syncs: Arc::new(Mutex::new(BTreeMap::new())), + state_sync_floor: None, + marshals: Arc::new(Mutex::new(BTreeMap::new())), + failures: Arc::new(HashSet::new()), + } + } + + pub(super) fn with_failures(mut self, failures: impl IntoIterator) -> Self { + self.failures = Arc::new(failures.into_iter().collect()); + self + } + + pub(super) const fn with_sharing_mode(mut self, sharing_mode: Mode) -> Self { + self.sharing_mode = sharing_mode; + self + } + + pub(super) const fn with_state_sync_floor(mut self, height: Height) -> Self { + self.state_sync_floor = Some(height); + self + } + + pub(super) const fn state_sync_floor(&self) -> Option { + self.state_sync_floor + } + + fn store(&self, public_key: &ed25519::PublicKey) -> MemorySecretStore { + let mut stores = self.stores.lock(); + match stores.entry(public_key.clone()) { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => { + let store = MemorySecretStore::default(); + if let Some(share) = self.initial.shares.get_value(public_key).cloned() { + store.seed_share(Epoch::zero(), share); + } + entry.insert(store.clone()); + store + } + } + } +} + +impl EngineDefinition for ReshareEngine { + type PublicKey = ed25519::PublicKey; + type Engine = ValidatorEngine; + type State = ValidatorState; + + fn participants(&self) -> Vec { + self.participants.clone() + } + + fn channels(&self) -> Vec<(u64, Quota)> { + vec![ + (VOTE_CHANNEL, TEST_QUOTA), + (CERTIFICATE_CHANNEL, TEST_QUOTA), + (RESOLVER_CHANNEL, TEST_QUOTA), + (BACKFILL_CHANNEL, TEST_QUOTA), + (BROADCAST_CHANNEL, TEST_QUOTA), + (QMDB_CHANNEL, TEST_QUOTA), + (DKG_CHANNEL, TEST_QUOTA), + (PROBE_CHANNEL, TEST_QUOTA), + (ANCHOR_BOUNDARY_CHANNEL, TEST_QUOTA), + ] + } + + async fn init(&self, ctx: InitContext<'_, Self::PublicKey>) -> (Self::Engine, Self::State) { + let InitContext { + context, + index, + delayed, + public_key, + oracle, + channels, + participants: _, + monitor, + } = ctx; + + let signer = self.signers[index].clone(); + let partition_prefix = format!("reshare-e2e-{index}"); + let page_cache = CacheRef::from_pooler(&context, PAGE_SIZE, PAGE_CACHE_SIZE); + + let mut channels = channels.into_iter(); + let vote_network = channels.next().unwrap(); + let certificate_network = channels.next().unwrap(); + let resolver_network = channels.next().unwrap(); + let backfill_network = channels.next().unwrap(); + let broadcast_network = channels.next().unwrap(); + let qmdb_network = channels.next().unwrap(); + let dkg_network = channels.next().unwrap(); + let probe_network = channels.next().unwrap(); + let anchor_boundary_network = channels.next().unwrap(); + + let (certificate_mux, certificate_mux_handle, certificate_backup) = Muxer::builder( + context.child("certificate_mux"), + certificate_network.0.clone(), + certificate_network.1, + 128, + ) + .with_backup() + .build(); + certificate_mux.start(); + let provider = DynamicProvider::new(); + let store = self.store(public_key); + self.initial.register_epoch_zero(&provider, &store).await; + + let resolver = marshal_resolver::init( + context.child("marshal_resolver"), + marshal_resolver::Config { + public_key: public_key.clone(), + peer_provider: oracle.manager(), + blocker: oracle.control(public_key.clone()), + mailbox_size: NZUsize!(100), + initial: Duration::from_secs(1), + timeout: Duration::from_secs(2), + fetch_retry_timeout: Duration::from_millis(100), + priority_requests: false, + priority_responses: false, + }, + backfill_network, + ); + + let broadcast_config = buffered::Config { + public_key: public_key.clone(), + mailbox_size: NZUsize!(100), + deque_size: 10, + priority: false, + codec_config: (), + peer_provider: oracle.manager(), + }; + let (broadcast_engine, buffer) = + buffered::Engine::new(context.child("broadcast"), broadcast_config); + broadcast_engine.start(broadcast_network); + + let finalizations_by_height = prunable::Archive::init( + context.child("finalizations_by_height"), + archive_config(&partition_prefix, "finalizations", page_cache.clone(), ()), + ) + .await + .expect("finalizations archive"); + let finalized_blocks = prunable::Archive::init( + context.child("finalized_blocks"), + archive_config(&partition_prefix, "blocks", page_cache.clone(), ()), + ) + .await + .expect("blocks archive"); + + let genesis = Block::genesis(self.participants[0].clone(), self.initial.info.clone()); + let (anchor_actor, anchor_mailbox) = anchor::Actor::new(anchor::Config { + context: context.child("anchor"), + manager: oracle.manager(), + peers: Set::from_iter_dedup(self.participants.iter().cloned()), + verifier: Scheme::certificate_verifier( + NAMESPACE, + *self.initial.info.output.public().public(), + ), + genesis: self.initial.info.clone(), + strategy: Sequential, + blocker: oracle.control(public_key.clone()), + blocks_per_epoch: EPOCH_LENGTH, + retry_timeout: NZDuration!(Duration::from_millis(500)), + mailbox_size: NZUsize!(100), + block_codec_config: (), + }); + let anchor_handle = anchor_actor.start(certificate_backup, anchor_boundary_network); + + let stateful_startup_context = context.child("stateful_startup"); + let mut plan = SyncPlan::init(&stateful_startup_context, partition_prefix.clone()).await; + let should_state_sync = plan.should_state_sync(delayed); + let anchor_artifact = if should_state_sync { + let artifact = anchor_mailbox.subscribe().await.expect("anchor stopped"); + provider.register( + artifact.epoch, + Scheme::verifier( + NAMESPACE, + artifact.info.output.players().clone(), + artifact.info.output.public().clone(), + ), + ); + Some(artifact) + } else { + None + }; + let minimum_probe_epoch = anchor_artifact + .as_ref() + .map_or_else(Epoch::zero, |artifact| artifact.epoch); + let (probe_actor, probe_mailbox) = Probe::new(ProbeConfig { + context: context.child("probe"), + provider: provider.clone(), + strategy: Sequential, + capacity: NZUsize!(100), + blocker: oracle.control(public_key.clone()), + minimum_epoch: minimum_probe_epoch, + retry_timeout: NZDuration!(Duration::from_millis(100)), + }); + let probe_handle = probe_actor.start(probe_network); + if should_state_sync { + let finalization = match self.state_sync_floor { + Some(height) => { + let source = self + .marshals + .lock() + .values() + .next() + .cloned() + .expect("state-sync floor source must be available"); + source + .get_finalization(height) + .await + .expect("configured state-sync floor must be finalized") + } + None => probe_mailbox.subscribe().await.expect("probe stopped"), + }; + plan = plan.with_floor(finalization); + } + let (marshal_actor, marshal, _) = MarshalActor::init( + context.child("marshal"), + finalizations_by_height, + finalized_blocks, + marshal::Config { + provider: provider.clone(), + epocher: FixedEpocher::new(EPOCH_LENGTH), + start: plan.marshal_start(genesis.clone()), + partition_prefix: partition_prefix.clone(), + mailbox_size: NZUsize!(100), + view_retention_timeout: ViewDelta::new(10), + prunable_items_per_section: NZU64!(10), + page_cache: page_cache.clone(), + replay_buffer: IO_BUFFER_SIZE, + key_write_buffer: IO_BUFFER_SIZE, + value_write_buffer: IO_BUFFER_SIZE, + block_codec_config: (), + max_repair: NZUsize!(10), + max_pending_acks: NZUsize!(1), + strategy: Sequential, + }, + ) + .await; + self.marshals + .lock() + .insert(public_key.clone(), marshal.clone()); + + let db_config = FixedConfig { + merkle_config: MmrJournalConfig { + journal_partition: format!("{partition_prefix}-qmdb-mmr-journal"), + metadata_partition: format!("{partition_prefix}-qmdb-mmr-metadata"), + items_per_blob: NZU64!(11), + write_buffer: IO_BUFFER_SIZE, + strategy: Sequential, + page_cache: page_cache.clone(), + }, + journal_config: FixedLogConfig { + partition: format!("{partition_prefix}-qmdb-log-journal"), + items_per_blob: NZU64!(7), + page_cache: page_cache.clone(), + write_buffer: IO_BUFFER_SIZE, + }, + translator: TwoCap, + init_cache_size: Some(NZUsize!(1024)), + }; + + let (qmdb_resolver_actor, qmdb_sync_resolver) = qmdb_resolver::Actor::new( + context.child("qmdb_resolver"), + qmdb_resolver::Config { + peer_provider: oracle.manager(), + blocker: oracle.control(public_key.clone()), + database: None, + mailbox_size: NZUsize!(100), + me: Some(public_key.clone()), + initial: Duration::from_secs(1), + timeout: Duration::from_secs(2), + fetch_retry_timeout: Duration::from_millis(100), + max_serve_ops: NZU64!(16), + priority_requests: false, + priority_responses: false, + }, + ); + let qmdb_handle = qmdb_resolver_actor.start(qmdb_network); + + let fence_epoch = anchor_artifact + .as_ref() + .map_or_else(Epoch::zero, |artifact| artifact.epoch); + let state_sync = anchor_artifact.map(|artifact| { + let floor = plan + .floor() + .cloned() + .expect("state-sync startup must have a probe floor"); + orchestrator::StateSync { artifact, floor } + }); + let registrar = TestRegistrar { + provider: provider.clone(), + events: self.registrations.clone(), + public_key: public_key.clone(), + }; + let (fence, gate) = Fence::new(fence_epoch); + let sync_floor: Option> = plan.floor().cloned(); + let state_sync_floor: Option = + sync_floor.as_ref().map(|floor| floor.proposal.payload); + let (reshare_actor, reshare_mailbox) = reshare::Actor::new( + context.child("reshare"), + reshare::Config { + signer: signer.clone(), + manager: oracle.manager(), + blocker: oracle.control(public_key.clone()), + participants_provider: ScheduleProvider { + schedule: self.schedule.clone(), + }, + secret_store: store, + strategy: Sequential, + registrar, + marshal: marshal.clone(), + state_sync_floor, + fence, + namespace: NAMESPACE, + sharing_mode: self.sharing_mode, + mailbox_size: NZUsize!(100), + partition_prefix: format!("{partition_prefix}-reshare"), + max_participants: MAX_PARTICIPANTS, + blocks_per_epoch: EPOCH_LENGTH, + batch_verifier: PhantomData::, + }, + ); + let dkg_network = ( + dkg_network.0, + FilteredReceiver::epochs(dkg_network.1, self.failures.clone()), + ); + let reshare_handle = reshare_actor.start(dkg_network); + + let (stateful_actor, stateful_mailbox) = StatefulActor::init( + context.child("stateful"), + StatefulConfig { + application: App { + genesis: genesis.clone(), + }, + db_config, + provider: (), + marshal: marshal.clone(), + mailbox_size: NZUsize!(100), + plan, + resolvers: qmdb_sync_resolver, + sync_config: SyncEngineConfig { + fetch_batch_size: NZU64!(16), + apply_batch_size: 64, + max_outstanding_requests: 8, + update_channel_size: NZUsize!(256), + max_retained_roots: 8, + }, + prune_config: None, + }, + ); + + // The reshare wrapper drives the payload for the stateful application. + let deferred = Deferred::new( + context.child("deferred"), + reshare::Application::new( + stateful_mailbox.clone(), + reshare_mailbox.clone(), + EPOCH_LENGTH, + ), + marshal.clone(), + FixedEpocher::new(EPOCH_LENGTH), + ); + + let (orchestrator_actor, orchestrator_mailbox) = orchestrator::Actor::new( + context.child("orchestrator"), + orchestrator::Config { + oracle: oracle.control(public_key.clone()), + manager: oracle.manager(), + provider: provider.clone(), + marshal: marshal.clone(), + application: deferred.clone(), + strategy: Sequential, + simplex: orchestrator::SimplexConfig { + elector: RoundRobin::::default(), + mailbox_size: NZUsize!(3), + replay_buffer: IO_BUFFER_SIZE, + write_buffer: IO_BUFFER_SIZE, + page_cache_page_size: PAGE_SIZE, + page_cache_pages: PAGE_CACHE_SIZE, + leader_timeout: Duration::from_secs(1), + certification_timeout: Duration::from_secs(2), + timeout_retry: Duration::from_millis(500), + fetch_timeout: Duration::from_secs(2), + fetch_concurrent: NZUsize!(3), + activity_timeout: ViewDelta::new(10), + skip_timeout: ViewDelta::new(5), + forwarding: ForwardingPolicy::Disabled, + }, + gate, + state_sync, + blocks_per_epoch: EPOCH_LENGTH, + muxer_size: 128, + mailbox_size: NZUsize!(100), + partition_prefix: format!("{partition_prefix}-orchestrator"), + }, + ); + let orchestrator_handle = + orchestrator_actor.start(vote_network, certificate_mux_handle, resolver_network); + + let reporters = Reporters::from(( + stateful_mailbox.clone(), + Reporters::from((orchestrator_mailbox, reshare_mailbox)), + )); + let marshal_handle = marshal_actor.start( + MonitorReporter::new(public_key.clone(), monitor, reporters), + buffer, + resolver, + ); + anchor_mailbox.attach(marshal.clone()); + probe_mailbox.attach(marshal.clone()); + if let Some(finalization) = sync_floor { + let block = marshal + .subscribe_by_commitment(finalization.proposal.payload, CommitmentFallback::Wait) + .await + .expect("sync floor block must be available"); + self.state_syncs + .lock() + .insert(public_key.clone(), block.height().get()); + } + let stateful_handle = stateful_actor.start(); + + ( + ValidatorEngine { + context, + handles: ValidatorHandles { + anchor: anchor_handle, + probe: probe_handle, + qmdb: qmdb_handle, + reshare: reshare_handle, + orchestrator: orchestrator_handle, + marshal: marshal_handle, + stateful: stateful_handle, + }, + }, + ValidatorState { + marshal, + registrations: self.registrations.clone(), + state_syncs: self.state_syncs.clone(), + public_key: public_key.clone(), + }, + ) + } + + fn start(engine: Self::Engine) -> Handle<()> { + let ValidatorEngine { context, handles } = engine; + context.spawn(move |_| handles.join()) + } +} + +#[derive(Clone)] +pub(super) struct ValidatorState { + pub(super) marshal: Marshal, + registrations: Arc>>>, + pub(super) state_syncs: Arc>>, + public_key: ed25519::PublicKey, +} + +impl PartialEq for ValidatorState { + fn eq(&self, other: &Self) -> bool { + self.public_key == other.public_key + } +} + +impl ProcessedHeight for ValidatorState { + async fn processed_height(&self) -> u64 { + self.marshal + .get_processed_height() + .await + .map_or(0, |height| height.get()) + } +} + +impl ValidatorState { + pub(super) fn public_key(&self) -> &ed25519::PublicKey { + &self.public_key + } + + pub(super) fn registrations(&self) -> Vec { + self.registrations + .lock() + .get(&self.public_key) + .cloned() + .unwrap_or_default() + } + + pub(super) fn state_sync_height(&self) -> Option { + self.state_syncs.lock().get(&self.public_key).copied() + } +} + +#[test] +fn restart_after_epoch_zero_pruning_does_not_reseed_share() { + let engine = ReshareEngine::new(); + let public_key = engine + .initial + .shares + .get(0) + .cloned() + .expect("epoch-zero player"); + + let store = engine.store(&public_key); + assert!(store.has_share(Epoch::zero())); + + futures::executor::block_on(async { + let mut pruned = store.clone(); + pruned.prune(Epoch::new(1)).await; + assert!(pruned.get_share(Epoch::zero()).await.is_none()); + + let restarted = engine.store(&public_key); + let mut restarted_view = restarted.clone(); + assert!(restarted_view.get_share(Epoch::zero()).await.is_none()); + + let provider = DynamicProvider::new(); + let role = engine + .initial + .register_epoch_zero(&provider, &restarted) + .await; + assert_eq!(role, RegistrationRole::Verifier); + }); +} + +fn archive_config( + prefix: &str, + name: &str, + page_cache: CacheRef, + codec_config: C, +) -> prunable::Config { + prunable::Config { + translator: TwoCap, + key_partition: format!("{prefix}-{name}-key"), + key_page_cache: page_cache, + value_partition: format!("{prefix}-{name}-value"), + compression: None, + codec_config, + items_per_section: NZU64!(10), + key_write_buffer: IO_BUFFER_SIZE, + value_write_buffer: IO_BUFFER_SIZE, + replay_buffer: IO_BUFFER_SIZE, + } +} + +fn empty_db_root() -> sha256::Digest { + Sha256Digest::from(hex!( + "ea6e0567a525372add5e4ef4d0600c18ed47fa5dd041a0ab0d25b60ea8c35978" + )) +} + +fn u64_to_digest(v: u64) -> sha256::Digest { + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&v.to_be_bytes()); + sha256::Digest::from(bytes) +} + +pub(super) fn final_height(epoch: u64) -> Height { + FixedEpocher::new(EPOCH_LENGTH) + .last(Epoch::new(epoch)) + .expect("test epoch should be supported") +} + +pub(super) fn height_round(height: Height) -> Round { + let info = FixedEpocher::new(EPOCH_LENGTH) + .containing(height) + .expect("test height should be supported"); + Round::new(info.epoch(), View::new(info.relative().get())) +} diff --git a/glue/src/dkg/tests/reshare/mod.rs b/glue/src/dkg/tests/reshare/mod.rs new file mode 100644 index 00000000000..ab17fd0308a --- /dev/null +++ b/glue/src/dkg/tests/reshare/mod.rs @@ -0,0 +1,467 @@ +use crate::simulate::{action::Crash, exit::ProcessedHeightAtLeast, plan::PlanBuilder}; +use commonware_consensus::types::{Epoch, Epocher, FixedEpocher}; +use commonware_cryptography::{bls12381::primitives::sharing::Mode, ed25519}; +use commonware_macros::{test_group, test_traced}; +use commonware_p2p::simulated::Link; +use std::time::Duration; + +mod harness; +use harness::{EPOCH_LENGTH, ReshareEngine, final_height, height_round}; + +mod properties; +use properties::{ + AllActiveProcessedHeight, AllNodesRecovered, BoundaryEpochInfos, BoundaryOutputMode, + EpochInfoContinuity, FailedCeremonyCarryOver, SchemesRegistered, SignerRegistered, + StateSyncMembership, StateSyncedAtHeight, StateSyncedSigner, +}; + +fn reshare_plan_with_boundary( + engine: ReshareEngine, + final_epoch: u64, + boundary: BoundaryEpochInfos, +) -> PlanBuilder { + let schedule = engine.schedule.clone(); + PlanBuilder::new(engine) + .seed(0) + .exit_condition(ProcessedHeightAtLeast::new(final_height(final_epoch).get())) + .property(SignerRegistered) + .property(boundary) + .property(EpochInfoContinuity::new(final_epoch + 1, schedule)) + .timeout(Duration::from_secs(60)) +} + +fn successful_reshare_plan(engine: ReshareEngine, final_epoch: u64) -> PlanBuilder { + reshare_plan_with_boundary( + engine, + final_epoch, + BoundaryEpochInfos::new(final_epoch + 1).with_no_reveals(), + ) +} + +fn successful_reshare_plan_with_schemes( + engine: ReshareEngine, + final_epoch: u64, +) -> PlanBuilder { + reshare_plan_with_schemes( + engine, + final_epoch, + BoundaryEpochInfos::new(final_epoch + 1).with_no_reveals(), + ) +} + +fn reshare_plan_with_schemes( + engine: ReshareEngine, + final_epoch: u64, + boundary: BoundaryEpochInfos, +) -> PlanBuilder { + let participants = engine.participants.clone(); + with_scheme_properties( + reshare_plan_with_boundary(engine, final_epoch, boundary), + participants, + Epoch::new(1), + Epoch::new(final_epoch + 1), + ) +} + +fn with_scheme_properties( + mut plan: PlanBuilder, + participants: Vec, + first: Epoch, + last: Epoch, +) -> PlanBuilder { + for epoch in first.get()..=last.get() { + plan = plan.property(SchemesRegistered::new( + participants.clone(), + Epoch::new(epoch), + )); + } + plan +} + +fn failed_carry_over_plan( + engine: ReshareEngine, + final_epoch: u64, + boundary: BoundaryEpochInfos, + failed_epochs: impl IntoIterator, +) -> PlanBuilder { + let schedule = (*engine.schedule).clone(); + let mut plan = reshare_plan_with_boundary(engine, final_epoch, boundary); + for epoch in failed_epochs { + plan = plan.property(FailedCeremonyCarryOver::new(epoch, schedule.clone())); + } + plan +} + +fn state_sync_next_player_plan( + engine: ReshareEngine, + final_epoch: u64, + delayed_index: usize, + next_player_epoch: Epoch, + signer_epoch: Epoch, + boundary: BoundaryEpochInfos, +) -> PlanBuilder { + let delayed = engine.participants[delayed_index].clone(); + let midpoint = FixedEpocher::new(EPOCH_LENGTH) + .midpoint(next_player_epoch) + .expect("test epoch should be supported"); + let configured_floor = engine.state_sync_floor(); + let sync_floor = configured_floor.unwrap_or_else(|| { + midpoint + .previous() + .expect("midpoint must have a sync floor") + }); + let max_sync_floor = configured_floor.unwrap_or_else(|| final_height(next_player_epoch.get())); + let registrations = engine.registrations.clone(); + let state_syncs = engine.state_syncs.clone(); + let schedule = engine.schedule.clone(); + reshare_plan_with_boundary(engine, final_epoch, boundary) + .crash(Crash::DelayRound { + participants: vec![delayed.clone()], + round: height_round(midpoint), + }) + .property(StateSyncMembership::new( + schedule, + delayed.clone(), + next_player_epoch, + )) + .property(StateSyncedAtHeight::new( + delayed.clone(), + sync_floor, + max_sync_floor, + state_syncs.clone(), + )) + .property(StateSyncedSigner::new( + delayed, + signer_epoch, + registrations, + state_syncs, + )) +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_deterministic() { + let first = successful_reshare_plan(ReshareEngine::new(), 1) + .run() + .unwrap() + .pop() + .unwrap() + .state; + let second = successful_reshare_plan(ReshareEngine::new(), 1) + .run() + .unwrap() + .pop() + .unwrap() + .state; + assert_eq!(first, second); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_deterministic_multiple_seeds() { + for seed in 0..3 { + let first = successful_reshare_plan(ReshareEngine::new(), 1) + .seed(seed) + .run() + .unwrap() + .pop() + .unwrap() + .state; + let second = successful_reshare_plan(ReshareEngine::new(), 1) + .seed(seed) + .run() + .unwrap() + .pop() + .unwrap() + .state; + assert_eq!(first, second); + } +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_single_participant_two_epochs() { + successful_reshare_plan_with_schemes(ReshareEngine::with_committee(1, 1), 1) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_rotates_participants() { + successful_reshare_plan_with_schemes(ReshareEngine::new(), 0) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_roots_of_unity_output() { + successful_reshare_plan_with_schemes( + ReshareEngine::new().with_sharing_mode(Mode::RootsOfUnity), + 0, + ) + .property(BoundaryOutputMode::new( + Epoch::zero(), + Mode::RootsOfUnity as u8, + )) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_multiple_epochs() { + successful_reshare_plan_with_schemes(ReshareEngine::new(), 4) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_multiple_epochs_rotating_subset() { + successful_reshare_plan_with_schemes(ReshareEngine::with_committee(7, 4), 4) + .timeout(Duration::from_secs(90)) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_failed_ceremony_carries_committee() { + failed_carry_over_plan( + ReshareEngine::new().with_failures([0]), + 0, + BoundaryEpochInfos::new(1).with_min_successes(0), + [Epoch::zero()], + ) + .timeout(Duration::from_secs(60)) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_consecutive_failed_ceremonies_carry_state() { + failed_carry_over_plan( + ReshareEngine::new().with_failures([0, 1]), + 2, + BoundaryEpochInfos::new(3) + .with_min_successes(1) + .with_no_reveals(), + [Epoch::zero(), Epoch::new(1)], + ) + .timeout(Duration::from_secs(90)) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_many_failed_ceremonies_carry_state() { + failed_carry_over_plan( + ReshareEngine::with_committees(8, vec![4, 5]).with_failures([0, 2, 3]), + 4, + BoundaryEpochInfos::new(5) + .with_min_successes(2) + .with_no_reveals(), + [Epoch::zero(), Epoch::new(2), Epoch::new(3)], + ) + .timeout(Duration::from_secs(120)) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_four_epochs_changing_size() { + successful_reshare_plan_with_schemes(ReshareEngine::with_committees(8, vec![4, 5, 6, 7, 4]), 4) + .run() + .unwrap(); +} + +fn crash_storm_plan( + engine: ReshareEngine, + final_epoch: u64, + crash: Crash, +) -> PlanBuilder { + let participants = engine.participants.clone(); + let target_height = final_height(final_epoch); + reshare_plan_with_schemes( + engine, + final_epoch, + BoundaryEpochInfos::new(final_epoch + 1), + ) + .crash(crash) + .exit_condition(AllActiveProcessedHeight::new( + target_height, + participants.len(), + )) + .property(AllNodesRecovered::new(participants)) + .timeout(Duration::from_secs(120)) +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_multiple_epochs_with_random_crashes() { + for seed in 0..3 { + crash_storm_plan( + ReshareEngine::new(), + 4, + Crash::Random { + frequency: Duration::from_secs(2), + downtime: Duration::from_millis(750), + count: 1, + }, + ) + .seed(seed) + .timeout(Duration::from_secs(180)) + .run() + .unwrap(); + } +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_multiple_epochs_with_many_random_crashes() { + crash_storm_plan( + ReshareEngine::new(), + 4, + Crash::Random { + frequency: Duration::from_secs(2), + downtime: Duration::from_millis(750), + count: 3, + }, + ) + .timeout(Duration::from_secs(360)) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_multiple_epochs_with_total_shutdown() { + crash_storm_plan( + ReshareEngine::new(), + 3, + Crash::Random { + frequency: Duration::from_secs(3), + downtime: Duration::from_secs(1), + count: 5, + }, + ) + .timeout(Duration::from_secs(180)) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_state_sync_epoch_first_next_player() { + let next_player_epoch = Epoch::new(1); + let state_sync_floor = FixedEpocher::new(EPOCH_LENGTH) + .first(next_player_epoch) + .expect("test epoch should be supported"); + state_sync_next_player_plan( + ReshareEngine::with_committee(6, 4).with_state_sync_floor(state_sync_floor), + 3, + 5, + next_player_epoch, + next_player_epoch.next(), + BoundaryEpochInfos::new(4), + ) + .timeout(Duration::from_secs(120)) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_state_sync_epoch_zero_next_player() { + let next_player_epoch = Epoch::zero(); + state_sync_next_player_plan( + ReshareEngine::new(), + 2, + 4, + next_player_epoch, + next_player_epoch.next(), + BoundaryEpochInfos::new(3).with_no_reveals(), + ) + .timeout(Duration::from_secs(120)) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_state_sync_next_player_after_failed_ceremony() { + let engine = ReshareEngine::with_committee(6, 4).with_failures([1]); + let next_player_epoch = Epoch::new(1); + let schedule = (*engine.schedule).clone(); + state_sync_next_player_plan( + engine, + 3, + 5, + next_player_epoch, + Epoch::new(3), + BoundaryEpochInfos::new(4) + .with_expected_failures([next_player_epoch.get()]) + .with_no_reveals(), + ) + .property(FailedCeremonyCarryOver::new(next_player_epoch, schedule)) + .timeout(Duration::from_secs(180)) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_state_sync_existing_player_after_failed_ceremony() { + let failed_epoch = Epoch::zero(); + let engine = ReshareEngine::new().with_failures([failed_epoch.get()]); + let delayed = engine.participants[1].clone(); + let midpoint = FixedEpocher::new(EPOCH_LENGTH) + .midpoint(failed_epoch) + .expect("test epoch should be supported"); + reshare_plan_with_boundary( + engine, + 1, + BoundaryEpochInfos::new(2).with_expected_failures([failed_epoch.get()]), + ) + .crash(Crash::DelayRound { + participants: vec![delayed.clone()], + round: height_round(midpoint), + }) + .property(SchemesRegistered::new(vec![delayed], failed_epoch.next())) + .timeout(Duration::from_secs(180)) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_rotating_subset() { + successful_reshare_plan_with_schemes(ReshareEngine::with_committee(7, 4), 0) + .run() + .unwrap(); +} + +#[test_group("slow")] +#[test_traced("INFO")] +fn reshare_e2e_lossy_network() { + reshare_plan_with_boundary( + ReshareEngine::new(), + 4, + BoundaryEpochInfos::new(5) + .with_min_successes(1) + .with_no_reveals(), + ) + .link(Link { + latency: Duration::from_millis(100), + jitter: Duration::from_millis(50), + success_rate: 0.7, + }) + .timeout(Duration::from_secs(720)) + .run() + .unwrap(); +} diff --git a/glue/src/dkg/tests/reshare/properties.rs b/glue/src/dkg/tests/reshare/properties.rs new file mode 100644 index 00000000000..792399ce699 --- /dev/null +++ b/glue/src/dkg/tests/reshare/properties.rs @@ -0,0 +1,780 @@ +use super::harness::{ + CommitteeSchedule, Registration, RegistrationRole, ValidatorState, final_height, +}; +use crate::{ + dkg::{ + ReshareBlock, + types::{EpochOutcome, Payload}, + }, + simulate::{ + exit::ExitCondition, processed::ProcessedHeight, property::Property, + tracker::ProgressTracker, + }, +}; +use commonware_codec::{Encode as _, FixedSize}; +use commonware_consensus::types::{Epoch, Height}; +use commonware_cryptography::{bls12381::primitives::variant::MinPk, ed25519, transcript::Summary}; +use commonware_utils::sync::Mutex; +use std::{collections::BTreeMap, future::Future, pin::Pin, sync::Arc}; + +#[derive(Clone)] +pub(super) struct AllActiveProcessedHeight { + required: Height, + participants: usize, +} + +impl AllActiveProcessedHeight { + pub(super) const fn new(required: Height, participants: usize) -> Self { + Self { + required, + participants, + } + } +} + +impl ExitCondition for AllActiveProcessedHeight { + fn name(&self) -> &str { + "all_active_processed_height" + } + + fn requires_polling(&self) -> bool { + true + } + + fn reached<'a>( + &'a self, + _tracker: &'a ProgressTracker, + states: &'a [&'a ValidatorState], + _target_count: usize, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + if states.len() != self.participants { + return Ok(false); + } + for state in states { + if state.processed_height().await < self.required.get() { + return Ok(false); + } + } + Ok(true) + }) + } +} + +#[derive(Clone)] +pub(super) struct SignerRegistered; + +impl Property for SignerRegistered { + fn name(&self) -> &str { + "signer_registered" + } + + fn check<'a>( + &'a self, + _tracker: &'a ProgressTracker, + states: &'a [&'a ValidatorState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let found = states.iter().any(|state| { + state + .registrations() + .iter() + .any(|registration| registration.role == RegistrationRole::Signer) + }); + if found { + Ok(()) + } else { + Err("no node registered a signing scheme".to_string()) + } + }) + } +} + +#[derive(Clone)] +pub(super) struct BoundaryEpochInfos { + epochs: u64, + no_reveals: bool, + min_successes: u64, + expected_failures: Vec, +} + +impl BoundaryEpochInfos { + pub(super) const fn new(epochs: u64) -> Self { + Self { + epochs, + no_reveals: false, + min_successes: epochs, + expected_failures: Vec::new(), + } + } + + pub(super) const fn with_no_reveals(mut self) -> Self { + self.no_reveals = true; + self + } + + pub(super) const fn with_min_successes(mut self, min_successes: u64) -> Self { + self.min_successes = min_successes; + self + } + + pub(super) fn with_expected_failures( + mut self, + failures: impl IntoIterator, + ) -> Self { + self.expected_failures = failures.into_iter().collect(); + self + } +} + +impl Property for BoundaryEpochInfos { + fn name(&self) -> &str { + "boundary_epoch_infos" + } + + fn check<'a>( + &'a self, + _tracker: &'a ProgressTracker, + states: &'a [&'a ValidatorState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + for state in states { + let mut checked = 0; + let mut successes = 0; + let mut expected_failures = 0; + let state_sync_height = state.state_sync_height(); + for epoch in 0..self.epochs { + let height = final_height(epoch); + if state_sync_height.is_some_and(|synced| height.get() < synced) { + continue; + } + checked += 1; + let expect_failure = self.expected_failures.contains(&epoch); + if expect_failure { + expected_failures += 1; + } + let Some(block) = state.marshal.get_block(height).await else { + return Err(format!( + "missing finalized boundary block at height {height}" + )); + }; + match block.payload() { + Some(Payload::EpochInfo(info)) if info.epoch == Epoch::new(epoch + 1) => { + if info.outcome == EpochOutcome::Success { + if expect_failure { + return Err(format!( + "boundary at height {height} succeeded, expected failure" + )); + } + successes += 1; + if self.no_reveals && !info.output.revealed().is_empty() { + return Err(format!( + "epoch {epoch} revealed {} shares", + info.output.revealed().len() + )); + } + continue; + } + if expect_failure { + continue; + } + if self.min_successes == self.epochs { + return Err(format!( + "boundary at height {height} carried epoch info {:?}", + info.outcome + )); + } + } + Some(Payload::EpochInfo(info)) => { + return Err(format!( + "boundary at height {height} carried epoch info for {}, expected {}", + info.epoch, + Epoch::new(epoch + 1) + )); + } + Some(_) => { + return Err(format!( + "boundary at height {height} carried non-epoch-info DKG payload" + )); + } + None => { + return Err(format!( + "boundary at height {height} carried no DKG payload" + )); + } + } + } + let required = if !self.expected_failures.is_empty() { + checked - expected_failures + } else if self.min_successes == self.epochs { + checked + } else { + self.min_successes.min(checked) + }; + if successes < required { + return Err(format!( + "observed {successes} successful epochs, expected at least {required}" + )); + } + } + Ok(()) + }) + } +} + +#[derive(Clone)] +pub(super) struct EpochInfoContinuity { + epochs: u64, + schedule: Arc, +} + +impl EpochInfoContinuity { + pub(super) const fn new(epochs: u64, schedule: Arc) -> Self { + Self { epochs, schedule } + } +} + +impl Property for EpochInfoContinuity { + fn name(&self) -> &str { + "epoch_info_continuity" + } + + fn check<'a>( + &'a self, + _tracker: &'a ProgressTracker, + states: &'a [&'a ValidatorState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + for epoch in 0..self.epochs { + let previous_height = Epoch::new(epoch) + .previous() + .map(|epoch| final_height(epoch.get())) + .unwrap_or(Height::zero()); + let previous = boundary_info(states, previous_height).await?; + + let height = final_height(epoch); + let info = boundary_info(states, height).await?; + let expected_epoch = Epoch::new(epoch + 1); + if info.epoch != expected_epoch { + return Err(format!( + "boundary at height {height} carried epoch info for {}, expected {expected_epoch}", + info.epoch + )); + } + if info.players != previous.next_players { + return Err(format!( + "boundary at height {height} players did not match previous next players" + )); + } + let expected_next_players = self.schedule.players(expected_epoch.next()); + if info.next_players != expected_next_players { + return Err(format!( + "boundary at height {height} next players did not match schedule for {}", + expected_epoch.next() + )); + } + + match info.outcome { + EpochOutcome::Success => { + if info.output.players() != &previous.players { + return Err(format!( + "successful boundary at height {height} output players did not match previous players" + )); + } + } + EpochOutcome::Failure => { + if info.output != previous.output { + return Err(format!( + "failed boundary at height {height} did not carry forward output" + )); + } + } + } + } + Ok(()) + }) + } +} + +async fn boundary_info( + states: &[&ValidatorState], + height: Height, +) -> Result, String> { + for state in states { + let Some(block) = state.marshal.get_block(height).await else { + continue; + }; + let Some(Payload::EpochInfo(info)) = block.payload() else { + return Err(format!( + "boundary at height {height} did not carry epoch info" + )); + }; + return Ok(info); + } + Err(format!( + "missing finalized boundary block at height {height}" + )) +} + +#[derive(Clone)] +pub(super) struct BoundaryOutputMode { + epoch: Epoch, + mode: u8, +} + +impl BoundaryOutputMode { + pub(super) const fn new(epoch: Epoch, mode: u8) -> Self { + Self { epoch, mode } + } +} + +impl Property for BoundaryOutputMode { + fn name(&self) -> &str { + "boundary_output_mode" + } + + fn check<'a>( + &'a self, + _tracker: &'a ProgressTracker, + states: &'a [&'a ValidatorState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let height = final_height(self.epoch.get()); + let info = boundary_info(states, height).await?; + + let encoded = info.output.encode(); + let Some(mode) = encoded.get(::SIZE).copied() else { + return Err("encoded output missing sharing mode".to_string()); + }; + if mode != self.mode { + return Err(format!( + "boundary at height {height} used sharing mode {mode}, expected {}", + self.mode + )); + } + Ok(()) + }) + } +} + +#[derive(Clone)] +pub(super) struct FailedCeremonyCarryOver { + epoch: Epoch, + schedule: CommitteeSchedule, +} + +impl FailedCeremonyCarryOver { + pub(super) const fn new(epoch: Epoch, schedule: CommitteeSchedule) -> Self { + Self { epoch, schedule } + } +} + +impl Property for FailedCeremonyCarryOver { + fn name(&self) -> &str { + "failed_ceremony_carry_over" + } + + fn check<'a>( + &'a self, + _tracker: &'a ProgressTracker, + states: &'a [&'a ValidatorState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + // The previous boundary may predate a node's state-sync floor and + // be absent from its marshal, so read it from whichever node + // retains it. + let previous_height = self + .epoch + .previous() + .map(|epoch| final_height(epoch.get())) + .unwrap_or(Height::zero()); + let previous = boundary_info(states, previous_height).await?; + + // The carry-over boundary itself must be visible to a state-synced + // node whose floor covers it: demand that node's view of the + // boundary rather than falling back to a peer that retains it. + let height = final_height(self.epoch.get()); + let synced = states.iter().find(|state| { + state + .state_sync_height() + .is_some_and(|floor| floor <= height.get()) + }); + let info = match synced { + Some(synced) => { + let Some(block) = synced.marshal.get_block(height).await else { + return Err(format!( + "state-synced node missing boundary block at height {height}" + )); + }; + let Some(Payload::EpochInfo(info)) = block.payload() else { + return Err(format!( + "boundary at height {height} did not carry epoch info" + )); + }; + info + } + None => boundary_info(states, height).await?, + }; + + let expected_epoch = self.epoch.next(); + if info.epoch != expected_epoch { + return Err(format!( + "boundary at height {height} carried epoch info for {}, expected {expected_epoch}", + info.epoch + )); + } + if info.outcome != EpochOutcome::Failure { + return Err(format!( + "boundary at height {height} carried {:?}, expected failure", + info.outcome + )); + } + if info.output != previous.output { + return Err("failed ceremony did not carry forward output".to_string()); + } + if info.players != previous.next_players { + return Err("failed ceremony did not advance to previous next players".to_string()); + } + let expected_next_players = self.schedule.players(expected_epoch.next()); + if info.next_players != expected_next_players { + return Err(format!( + "failed ceremony did not refresh next players for {}", + expected_epoch.next() + )); + } + + for state in states { + let expected = if previous + .output + .players() + .position(state.public_key()) + .is_some() + { + RegistrationRole::Signer + } else { + RegistrationRole::Verifier + }; + let registrations = state.registrations(); + let registered = registrations.iter().any(|registration| { + registration.epoch == expected_epoch && registration.role == expected + }); + if !registered { + return Err(format!( + "node {} did not register {expected:?} for carried epoch {expected_epoch}: {registrations:?}", + state.public_key() + )); + } + } + + Ok(()) + }) + } +} + +#[derive(Clone)] +pub(super) struct AllNodesRecovered { + public_keys: Vec, +} + +impl AllNodesRecovered { + pub(super) fn new(public_keys: Vec) -> Self { + Self { public_keys } + } +} + +impl Property for AllNodesRecovered { + fn name(&self) -> &str { + "all_nodes_recovered" + } + + fn check<'a>( + &'a self, + tracker: &'a ProgressTracker, + states: &'a [&'a ValidatorState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + if states.len() != self.public_keys.len() { + return Err(format!( + "active states {}, expected {}", + states.len(), + self.public_keys.len() + )); + } + if tracker.tracked_count() != self.public_keys.len() { + return Err(format!( + "tracker saw {} nodes, expected {}", + tracker.tracked_count(), + self.public_keys.len() + )); + } + for public_key in &self.public_keys { + let recovered = states.iter().any(|state| state.public_key() == public_key); + if !recovered { + return Err(format!("node {public_key} was not active at shutdown")); + } + } + Ok(()) + }) + } +} + +#[derive(Clone)] +pub(super) struct SchemesRegistered { + public_keys: Vec, + epoch: Epoch, +} + +impl SchemesRegistered { + pub(super) fn new(public_keys: Vec, epoch: Epoch) -> Self { + Self { public_keys, epoch } + } +} + +impl Property for SchemesRegistered { + fn name(&self) -> &str { + "schemes_registered" + } + + fn check<'a>( + &'a self, + _tracker: &'a ProgressTracker, + states: &'a [&'a ValidatorState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let Some(ceremony_epoch) = self.epoch.previous() else { + return Err("epoch zero has no reshare output".to_string()); + }; + let height = final_height(ceremony_epoch.get()); + let Some(reference) = states.first() else { + return Err("no active validator states".to_string()); + }; + let Some(block) = reference.marshal.get_block(height).await else { + return Err(format!( + "missing finalized boundary block at height {height}" + )); + }; + let Some(Payload::EpochInfo(info)) = block.payload() else { + return Err(format!( + "boundary at height {height} did not carry epoch info" + )); + }; + if info.epoch != self.epoch { + return Err(format!( + "boundary at height {height} carried epoch info for {}, expected {}", + info.epoch, self.epoch + )); + } + for public_key in &self.public_keys { + let expected = if info.output.players().position(public_key).is_some() { + RegistrationRole::Signer + } else { + RegistrationRole::Verifier + }; + let Some(registrations) = states + .iter() + .find(|state| state.public_key() == public_key) + .map(|state| state.registrations()) + else { + return Err(format!("node {public_key} was not active at shutdown")); + }; + let registered = registrations.iter().any(|registration| { + registration.epoch == self.epoch && registration.role == expected + }); + if !registered { + return Err(format!( + "node {public_key} did not register {expected:?} for epoch {}: {registrations:?}", + self.epoch + )); + } + } + Ok(()) + }) + } +} + +#[derive(Clone)] +pub(super) struct StateSyncedSigner { + public_key: ed25519::PublicKey, + min_epoch: Epoch, + registrations: Arc>>>, + state_syncs: Arc>>, +} + +impl StateSyncedSigner { + pub(super) fn new( + public_key: ed25519::PublicKey, + min_epoch: Epoch, + registrations: Arc>>>, + state_syncs: Arc>>, + ) -> Self { + Self { + public_key, + min_epoch, + registrations, + state_syncs, + } + } +} + +impl Property for StateSyncedSigner { + fn name(&self) -> &str { + "state_synced_signer" + } + + fn check<'a>( + &'a self, + _tracker: &'a ProgressTracker, + _states: &'a [&'a ValidatorState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let state_syncs = self.state_syncs.lock(); + let Some(height) = state_syncs.get(&self.public_key).copied() else { + let keys = state_syncs.keys().cloned().collect::>(); + return Err(format!( + "node {} did not state sync, recorded syncs: {keys:?}", + self.public_key + )); + }; + let signed = self + .registrations + .lock() + .get(&self.public_key) + .into_iter() + .flatten() + .any(|registration| { + registration.role == RegistrationRole::Signer + && registration.epoch >= self.min_epoch + }); + if signed { + Ok(()) + } else { + Err(format!( + "node {} state synced at height {height} but never registered as signer at or after epoch {}", + self.public_key, self.min_epoch + )) + } + }) + } +} + +#[derive(Clone)] +pub(super) struct StateSyncedAtHeight { + public_key: ed25519::PublicKey, + min_height: Height, + max_height: Height, + state_syncs: Arc>>, +} + +impl StateSyncedAtHeight { + pub(super) const fn new( + public_key: ed25519::PublicKey, + min_height: Height, + max_height: Height, + state_syncs: Arc>>, + ) -> Self { + Self { + public_key, + min_height, + max_height, + state_syncs, + } + } +} + +impl Property for StateSyncedAtHeight { + fn name(&self) -> &str { + "state_synced_at_height" + } + + fn check<'a>( + &'a self, + _tracker: &'a ProgressTracker, + _states: &'a [&'a ValidatorState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let state_syncs = self.state_syncs.lock(); + let Some(height) = state_syncs.get(&self.public_key).copied() else { + return Err(format!("node {} did not state sync", self.public_key)); + }; + if height < self.min_height.get() || height > self.max_height.get() { + return Err(format!( + "node {} state synced at height {height}, expected {}..={}", + self.public_key, self.min_height, self.max_height + )); + } + Ok(()) + }) + } +} + +#[derive(Clone)] +pub(super) struct StateSyncMembership { + schedule: Arc, + public_key: ed25519::PublicKey, + next_player_epoch: Epoch, +} + +impl StateSyncMembership { + pub(super) fn new( + schedule: Arc, + public_key: ed25519::PublicKey, + next_player_epoch: Epoch, + ) -> Self { + Self { + schedule, + public_key, + next_player_epoch, + } + } +} + +impl Property for StateSyncMembership { + fn name(&self) -> &str { + "state_sync_membership" + } + + fn check<'a>( + &'a self, + _tracker: &'a ProgressTracker, + _states: &'a [&'a ValidatorState], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let player_epoch = self.next_player_epoch.next(); + if let Some(previous) = self.next_player_epoch.previous() + && self + .schedule + .players(previous) + .position(&self.public_key) + .is_some() + { + return Err(format!( + "node {} was present before state-sync epoch {}", + self.public_key, self.next_player_epoch + )); + } + if self + .schedule + .players(self.next_player_epoch) + .position(&self.public_key) + .is_some() + { + return Err(format!( + "node {} was a player in state-sync epoch {}", + self.public_key, self.next_player_epoch + )); + } + if self + .schedule + .players(player_epoch) + .position(&self.public_key) + .is_none() + { + return Err(format!( + "node {} was not a next player in epoch {}", + self.public_key, self.next_player_epoch + )); + } + Ok(()) + }) + } +} diff --git a/glue/src/dkg/types.rs b/glue/src/dkg/types.rs new file mode 100644 index 00000000000..5f2019868b2 --- /dev/null +++ b/glue/src/dkg/types.rs @@ -0,0 +1,608 @@ +//! Shared types for the DKG module. + +use crate::dkg::reshare::MAX_SUPPORTED_MODE; +use bytes::{Buf, BufMut}; +use commonware_codec::{EncodeSize, Error as CodecError, RangeCfg, Read, ReadExt, Write}; +use commonware_consensus::types::Epoch; +use commonware_cryptography::{ + PublicKey, Signer, + bls12381::{ + dkg::feldman_desmedt::{DealerPrivMsg, DealerPubMsg, Output, PlayerAck, SignedDealerLog}, + primitives::{group::Share, sharing::Sharing, variant::Variant}, + }, +}; +use commonware_p2p::TrackedPeers; +use commonware_utils::{Faults as _, N3f1, ordered::Set}; +use std::num::{NonZeroU32, NonZeroU64}; +use thiserror::Error; + +/// Information required to construct an epoch-scoped threshold scheme that may +/// or may not be capable of signing messages. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SchemeInfo { + /// Information required for constructing a verifier scheme. + Verifier { + /// The participants. + participants: Set

, + /// The public group polynomial. + sharing: Sharing, + }, + /// Information required for constructing a signer scheme. + Signer { + /// The participants. + participants: Set

, + /// The public group polynomial. + sharing: Sharing, + /// A BLS [`Share`]. + share: Share, + }, +} + +/// Result of a completed DKG/reshare epoch. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum EpochOutcome { + /// The epoch produced a new public output. + Success, + /// The epoch failed and carried the previous public state forward. + Failure, +} + +impl Write for EpochOutcome { + fn write(&self, writer: &mut impl BufMut) { + let tag = match self { + Self::Success => 0u8, + Self::Failure => 1u8, + }; + tag.write(writer); + } +} + +impl EncodeSize for EpochOutcome { + fn encode_size(&self) -> usize { + 1 + } +} + +impl Read for EpochOutcome { + type Cfg = (); + + fn read_cfg(reader: &mut impl Buf, _: &Self::Cfg) -> Result { + match u8::read(reader)? { + 0 => Ok(Self::Success), + 1 => Ok(Self::Failure), + n => Err(CodecError::InvalidEnum(n)), + } + } +} + +/// Participants for a DKG/reshare epoch. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Participants { + /// Peers that distribute dealings in this epoch. + pub dealers: Set

, + /// Peers that receive shares in this epoch. + pub players: Set

, + /// Players of the next epoch, tracked early for connectivity. + pub next_players: Set

, +} + +/// Errors produced while validating DKG/reshare participants. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum ParticipantsError { + /// No dealers were provided. + #[error("dealers must not be empty")] + EmptyDealers, + /// No players were provided. + #[error("players must not be empty")] + EmptyPlayers, + /// A participant set exceeds the configured maximum. + #[error("too many participants: {actual} > {max}")] + TooManyParticipants { actual: usize, max: usize }, + /// Round-zero reshare dealers differ from the previous output players. + #[error("round-zero reshare dealers must equal previous output players")] + InitialReshareDealers, + /// A later reshare dealer does not own a previous share. + #[error("reshare dealer is not a previous player")] + UnknownReshareDealer, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct EpochCapacityError { + available: u64, + required: u64, +} + +impl Participants

{ + /// Builds the peer set used by the DKG channel. + /// + /// Dealers are the primary tracked peers because they send protocol data in the + /// current round. Current and next players are tracked as secondary peers so the + /// actor keeps enough connectivity to receive its own messages and prepare the + /// next epoch without allowing next players to act as dealers. + pub fn tracked_peers(&self) -> TrackedPeers

{ + TrackedPeers::new( + self.dealers.clone(), + Set::from_iter_dedup(self.players.iter().chain(self.next_players.iter()).cloned()), + ) + } + + /// Checks that a participant snapshot is usable for the requested reshare round. + /// + /// Reshare requires non-empty dealer and player sets, caps every participant set + /// at `max_participants`, and verifies that reshare dealers are authorized by the + /// previous epoch output. Round zero must start from exactly the previous player + /// set. Later rounds may use any subset of previous players as dealers. + pub fn validate( + &self, + max_participants: NonZeroU32, + previous: Option<&Output>, + round: u64, + ) -> Result<(), ParticipantsError> { + if self.dealers.is_empty() { + return Err(ParticipantsError::EmptyDealers); + } + if self.players.is_empty() { + return Err(ParticipantsError::EmptyPlayers); + } + + let max = max_participants.get() as usize; + for actual in [ + self.dealers.len(), + self.players.len(), + self.next_players.len(), + ] { + if actual > max { + return Err(ParticipantsError::TooManyParticipants { actual, max }); + } + } + + let Some(previous) = previous else { + return Ok(()); + }; + + if round == 0 { + if &self.dealers != previous.players() { + return Err(ParticipantsError::InitialReshareDealers); + } + return Ok(()); + } + + if self + .dealers + .iter() + .any(|dealer| previous.players().position(dealer).is_none()) + { + return Err(ParticipantsError::UnknownReshareDealer); + } + + Ok(()) + } + + pub(crate) fn validate_epoch_capacity( + &self, + blocks_per_epoch: NonZeroU64, + previous: Option<&Output>, + ) -> Result<(), EpochCapacityError> { + let available = dealer_log_slots(blocks_per_epoch); + let required = self.required_dealer_logs(previous); + if available < required { + return Err(EpochCapacityError { + available, + required, + }); + } + Ok(()) + } + + fn required_dealer_logs(&self, previous: Option<&Output>) -> u64 { + let dealer_quorum = u64::from(N3f1::quorum(self.dealers.len())); + let previous_quorum = previous + .map(|previous| u64::from(previous.quorum::())) + .unwrap_or_default(); + dealer_quorum.max(previous_quorum) + } +} + +const fn dealer_log_slots(blocks_per_epoch: NonZeroU64) -> u64 { + let blocks = blocks_per_epoch.get(); + // Shorter epochs do not leave a usable dealing-to-inclusion window. + if blocks < 4 { + return 0; + } + blocks.saturating_sub(blocks / 2 + 1) +} + +impl Write for Participants

{ + fn write(&self, writer: &mut impl BufMut) { + self.dealers.write(writer); + self.players.write(writer); + self.next_players.write(writer); + } +} + +impl EncodeSize for Participants

{ + fn encode_size(&self) -> usize { + self.dealers.encode_size() + self.players.encode_size() + self.next_players.encode_size() + } +} + +impl Read for Participants

{ + /// Maximum number of participants accepted in any single set. + type Cfg = NonZeroU32; + + fn read_cfg(reader: &mut impl Buf, max: &Self::Cfg) -> Result { + let cfg = (RangeCfg::new(0..=max.get() as usize), ()); + Ok(Self { + dealers: Set::read_cfg(reader, &cfg)?, + players: Set::read_cfg(reader, &cfg)?, + next_players: Set::read_cfg(reader, &cfg)?, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use commonware_cryptography::{ + bls12381::{ + dkg::feldman_desmedt::deal, + primitives::{sharing::Mode, variant::MinPk}, + }, + ed25519, + }; + use commonware_utils::{NZU64, TestRng}; + + fn keys(count: u64) -> Set { + Set::from_iter_dedup( + (0..count).map(|seed| ed25519::PrivateKey::from_seed(seed).public_key()), + ) + } + + fn participants(count: u64) -> Participants { + let keys = keys(count); + Participants { + dealers: keys.clone(), + players: keys, + next_players: Set::default(), + } + } + + #[test] + fn epoch_capacity_rejects_insufficient_bootstrap_slots() { + assert_eq!( + participants(2).validate_epoch_capacity::(NZU64!(4), None), + Err(EpochCapacityError { + available: 1, + required: 2, + }) + ); + } + + #[test] + fn epoch_capacity_accepts_exact_bootstrap_slots() { + assert!( + participants(2) + .validate_epoch_capacity::(NZU64!(5), None) + .is_ok() + ); + } + + #[test] + fn epoch_capacity_rejects_short_epoch_with_single_dealer() { + assert_eq!( + participants(1).validate_epoch_capacity::(NZU64!(3), None), + Err(EpochCapacityError { + available: 0, + required: 1, + }) + ); + } + + #[test] + fn epoch_capacity_rejects_insufficient_reshare_slots() { + let players = keys(4); + let (previous, _) = + deal::(TestRng::new(0), Mode::NonZeroCounter, players.clone()) + .expect("trusted deal"); + + assert_eq!( + Participants { + dealers: players.clone(), + players, + next_players: Set::default(), + } + .validate_epoch_capacity(NZU64!(6), Some(&previous)), + Err(EpochCapacityError { + available: 2, + required: 3, + }) + ); + } + + #[test] + fn epoch_capacity_accepts_exact_reshare_slots() { + let players = keys(4); + let (previous, _) = + deal::(TestRng::new(1), Mode::NonZeroCounter, players.clone()) + .expect("trusted deal"); + + assert!( + Participants { + dealers: players.clone(), + players, + next_players: Set::default(), + } + .validate_epoch_capacity(NZU64!(7), Some(&previous)) + .is_ok() + ); + } +} + +#[cfg(feature = "arbitrary")] +impl arbitrary::Arbitrary<'_> for Participants

+where + P: for<'a> arbitrary::Arbitrary<'a>, +{ + fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { + Ok(Self { + dealers: u.arbitrary()?, + players: u.arbitrary()?, + next_players: u.arbitrary()?, + }) + } +} + +/// Canonical public epoch artifact. +/// +/// This is the public truth needed to start an epoch: the latest public output +/// and the participant sets not already carried by that output. +/// The genesis block carries the [`EpochInfo`] for epoch 0; the final block of +/// each epoch carries the [`EpochInfo`] for the following epoch. The reshare +/// actor never invents this; it reads it back from finalized block ancestry. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EpochInfo { + /// Whether or not the reshare ceremony in this epoch was successful. + pub outcome: EpochOutcome, + /// Epoch this artifact describes. + pub epoch: Epoch, + /// Latest public DKG output. + pub output: Output, + /// Peers that receive shares in this epoch. + pub players: Set

, + /// Players of the next epoch, tracked early for connectivity. + pub next_players: Set

, +} + +impl EpochInfo { + /// Reconstructs the complete participant snapshot for this epoch. + pub fn participants(&self) -> Participants

{ + Participants { + dealers: self.output.players().clone(), + players: self.players.clone(), + next_players: self.next_players.clone(), + } + } +} + +impl Write for EpochInfo { + fn write(&self, buf: &mut impl BufMut) { + self.outcome.write(buf); + self.epoch.write(buf); + self.output.write(buf); + self.players.write(buf); + self.next_players.write(buf); + } +} + +impl EncodeSize for EpochInfo { + fn encode_size(&self) -> usize { + self.outcome.encode_size() + + self.epoch.encode_size() + + self.output.encode_size() + + self.players.encode_size() + + self.next_players.encode_size() + } +} + +impl Read for EpochInfo { + /// Maximum number of participants accepted in any single set. + type Cfg = NonZeroU32; + + fn read_cfg(buf: &mut impl Buf, max: &Self::Cfg) -> Result { + Ok(Self { + outcome: EpochOutcome::read(buf)?, + epoch: Epoch::read(buf)?, + output: Output::::read_cfg(buf, &(*max, MAX_SUPPORTED_MODE))?, + players: Set::read_cfg(buf, &(RangeCfg::new(0..=max.get() as usize), ()))?, + next_players: Set::read_cfg(buf, &(RangeCfg::new(0..=max.get() as usize), ()))?, + }) + } +} + +#[cfg(feature = "arbitrary")] +impl arbitrary::Arbitrary<'_> for EpochInfo +where + P: for<'a> arbitrary::Arbitrary<'a>, + Output: for<'a> arbitrary::Arbitrary<'a>, +{ + fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { + Ok(Self { + outcome: u.arbitrary()?, + epoch: u.arbitrary()?, + output: u.arbitrary()?, + players: u.arbitrary()?, + next_players: u.arbitrary()?, + }) + } +} + +/// A public artifact published by the reshare actor into a block. +/// +/// During the dealing and inclusion window of an epoch the actor publishes +/// finalized dealer logs. The final block of an epoch instead carries the +/// canonical [`EpochInfo`] for the following epoch. +#[allow(clippy::large_enum_variant)] +pub enum Payload { + /// A finalized signed dealer log for inclusion mid-epoch. + DealerLog(SignedDealerLog), + /// The canonical public epoch artifact for the next epoch, carried by the + /// final block of the current epoch. + EpochInfo(EpochInfo), +} + +impl Clone for Payload { + fn clone(&self) -> Self { + match self { + Self::DealerLog(log) => Self::DealerLog(log.clone()), + Self::EpochInfo(info) => Self::EpochInfo(info.clone()), + } + } +} + +impl PartialEq for Payload { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::DealerLog(a), Self::DealerLog(b)) => a == b, + (Self::EpochInfo(a), Self::EpochInfo(b)) => a == b, + _ => false, + } + } +} + +impl Eq for Payload {} + +impl Write for Payload { + fn write(&self, writer: &mut impl BufMut) { + match self { + Self::DealerLog(log) => { + 0u8.write(writer); + log.write(writer); + } + Self::EpochInfo(info) => { + 1u8.write(writer); + info.write(writer); + } + } + } +} + +impl EncodeSize for Payload { + fn encode_size(&self) -> usize { + 1 + match self { + Self::DealerLog(log) => log.encode_size(), + Self::EpochInfo(info) => info.encode_size(), + } + } +} + +impl Read for Payload { + /// Maximum number of participants accepted in decoded artifacts. + type Cfg = NonZeroU32; + + fn read_cfg(reader: &mut impl Buf, max: &Self::Cfg) -> Result { + match u8::read(reader)? { + 0 => Ok(Self::DealerLog(SignedDealerLog::read_cfg(reader, max)?)), + 1 => Ok(Self::EpochInfo(EpochInfo::read_cfg(reader, max)?)), + n => Err(CodecError::InvalidEnum(n)), + } + } +} + +#[cfg(feature = "arbitrary")] +impl arbitrary::Arbitrary<'_> for Payload +where + SignedDealerLog: for<'a> arbitrary::Arbitrary<'a>, + EpochInfo: for<'a> arbitrary::Arbitrary<'a>, +{ + fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { + Ok(if u.arbitrary::()? { + Self::DealerLog(u.arbitrary()?) + } else { + Self::EpochInfo(u.arbitrary()?) + }) + } +} + +/// Wire message type for DKG protocol communication. +pub enum Message { + /// A dealer message containing public and private components for a player. + Dealer(DealerPubMsg, DealerPrivMsg), + /// A player acknowledgment sent back to a dealer. + Ack(PlayerAck

), +} + +impl Write for Message { + fn write(&self, writer: &mut impl BufMut) { + match self { + Self::Dealer(pub_msg, priv_msg) => { + 0u8.write(writer); + pub_msg.write(writer); + priv_msg.write(writer); + } + Self::Ack(ack) => { + 1u8.write(writer); + ack.write(writer); + } + } + } +} + +impl EncodeSize for Message { + fn encode_size(&self) -> usize { + 1 + match self { + Self::Dealer(pub_msg, priv_msg) => pub_msg.encode_size() + priv_msg.encode_size(), + Self::Ack(ack) => ack.encode_size(), + } + } +} + +impl Read for Message { + type Cfg = NonZeroU32; + + fn read_cfg(reader: &mut impl Buf, cfg: &Self::Cfg) -> Result { + let tag = u8::read(reader)?; + match tag { + 0 => { + let pub_msg = DealerPubMsg::read_cfg(reader, cfg)?; + let priv_msg = DealerPrivMsg::read(reader)?; + Ok(Self::Dealer(pub_msg, priv_msg)) + } + 1 => { + let ack = PlayerAck::read(reader)?; + Ok(Self::Ack(ack)) + } + n => Err(CodecError::InvalidEnum(n)), + } + } +} + +#[cfg(feature = "arbitrary")] +impl arbitrary::Arbitrary<'_> for Message +where + DealerPubMsg: for<'a> arbitrary::Arbitrary<'a>, + PlayerAck

: for<'a> arbitrary::Arbitrary<'a>, +{ + fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { + Ok(if u.arbitrary::()? { + Self::Dealer(u.arbitrary()?, u.arbitrary()?) + } else { + Self::Ack(u.arbitrary()?) + }) + } +} + +#[cfg(all(test, feature = "arbitrary"))] +mod conformance { + use super::*; + use commonware_codec::conformance::CodecConformance; + use commonware_cryptography::{bls12381::primitives::variant::MinSig, ed25519}; + + commonware_conformance::conformance_tests! { + CodecConformance, + CodecConformance>, + CodecConformance> => 8192, + CodecConformance> => 8192, + CodecConformance>, + } +} diff --git a/glue/src/lib.rs b/glue/src/lib.rs index 62fe1eca6db..6a342ea327a 100644 --- a/glue/src/lib.rs +++ b/glue/src/lib.rs @@ -5,6 +5,7 @@ )] commonware_macros::stability_scope!(ALPHA { + pub mod dkg; pub mod stateful; #[cfg(any(test, feature = "test-utils"))] diff --git a/glue/src/simulate/action.rs b/glue/src/simulate/action.rs index 4972b37c2bc..e6c49336f87 100644 --- a/glue/src/simulate/action.rs +++ b/glue/src/simulate/action.rs @@ -1,5 +1,6 @@ //! Simulation action types for testing. +use commonware_consensus::types::Round; use commonware_cryptography::PublicKey; use commonware_p2p::simulated::Link; use commonware_runtime::deterministic; @@ -19,12 +20,12 @@ pub enum Crash { count: usize, }, - /// Delay some validators from starting until after N finalizations. - Delay { - /// Number of validators to delay. - count: usize, - /// Number of finalizations before starting delayed validators. - after: u64, + /// Delay specific validators until the given round is observed. + DelayRound { + /// Validators to delay. + participants: Vec

, + /// Round to wait for before starting delayed validators. + round: Round, }, /// Time-indexed action schedule for precise control. diff --git a/glue/src/simulate/plan.rs b/glue/src/simulate/plan.rs index 7ce39673225..6f274fb16b9 100644 --- a/glue/src/simulate/plan.rs +++ b/glue/src/simulate/plan.rs @@ -47,7 +47,7 @@ pub struct PlanResult { /// Number of scheduled actions that were applied. pub scheduled_actions: u64, - /// Whether delayed validators were started (if Delay was configured). + /// Whether delayed validators were started. pub delayed_started: bool, } @@ -186,12 +186,12 @@ impl PlanBuilder { pub fn crash(mut self, crash: Crash) -> Self { match crash { - Crash::Delay { .. } => assert!( + Crash::DelayRound { .. } => assert!( !self .crashes .iter() - .any(|crash| matches!(crash, Crash::Delay { .. })), - "only one Crash::Delay strategy may be configured" + .any(|crash| matches!(crash, Crash::DelayRound { .. })), + "only one delay strategy may be configured" ), Crash::Random { .. } => assert!( !self @@ -309,13 +309,19 @@ impl Plan { }) } - fn delay_crash(&self) -> Option<(usize, u64)> { - self.crashes.iter().find_map(|crash| match crash { - Crash::Delay { count, after } => Some((*count, *after)), - _ => None, + fn delay_reached(&self, tracker: &ProgressTracker) -> bool { + self.crashes.iter().any(|crash| match crash { + Crash::DelayRound { round, .. } => tracker.max_round().is_some_and(|max| max >= *round), + _ => false, }) } + fn has_delay(&self) -> bool { + self.crashes + .iter() + .any(|crash| matches!(crash, Crash::DelayRound { .. })) + } + fn random_crash(&self) -> Option<(Duration, Duration, usize)> { self.crashes.iter().find_map(|crash| match crash { Crash::Random { @@ -336,11 +342,15 @@ impl Plan { /// Determine which participants should be delayed at startup. fn delayed_participants(&self) -> HashSet { - if let Some((count, _)) = self.delay_crash() { - self.participants.iter().take(count).cloned().collect() - } else { - HashSet::new() - } + self.crashes + .iter() + .find_map(|crash| match crash { + Crash::DelayRound { participants, .. } => { + Some(participants.iter().cloned().collect()) + } + _ => None, + }) + .unwrap_or_default() } /// Check post-run properties, log completion, and build the result. @@ -403,7 +413,7 @@ impl Plan { simulated::Config { max_size: self.max_message_size, disconnect_on_block: true, - tracked_peer_sets: NZUsize!(3), + tracked_peer_sets: NZUsize!(1), }, ); network.start(); @@ -553,10 +563,7 @@ impl Plan { } // Start delayed validators after enough progress - if !delayed_started - && let Some((_, after)) = self.delay_crash() - && tracker.min_view() >= after - { + if !delayed_started && !delayed.is_empty() && self.delay_reached(&tracker) { info!(target: "simulator", "starting delayed participants"); for pk in &delayed { team.start_one(&ctx, &oracle, pk.clone(), monitor_tx.clone(), true) @@ -659,11 +666,11 @@ impl Plan { ); } - if self.delay_crash().is_some() { + if self.has_delay() { assert!( r.delayed_started, - "Crash::Delay configured but delayed validators were never started. \ - Increase required_finalizations or decrease the `after` threshold." + "delay configured but delayed validators were never started. \ + Increase required_finalizations or decrease the delay threshold." ); } } @@ -762,7 +769,7 @@ impl Plan { #[cfg(test)] mod tests { use super::*; - use commonware_consensus::types::View; + use commonware_consensus::types::{Epoch, Round, View}; use commonware_cryptography::{Signer as _, ed25519}; use commonware_runtime::{Clock, Handle, Quota, Spawner}; use std::{ @@ -868,7 +875,7 @@ mod tests { let _ = monitor .send(FinalizationUpdate { pk: pk.clone(), - view: View::new(view), + round: Round::new(Epoch::zero(), View::new(view)), block_digest: vec![view as u8], }) .await; @@ -915,7 +922,7 @@ mod tests { let _ = monitor .send(FinalizationUpdate { pk, - view: View::new(1), + round: Round::new(Epoch::zero(), View::new(1)), block_digest: vec![1], }) .await; @@ -1030,10 +1037,15 @@ mod tests { jitter: Duration::from_millis(0), success_rate: 1.0, }; - let result = PlanBuilder::new(FinalizingEngine::new(2, Duration::from_millis(100), 2)) + let engine = FinalizingEngine::new(2, Duration::from_millis(100), 2); + let delayed = engine.participants[0].clone(); + let result = PlanBuilder::new(engine) .required_finalizations(2) .timeout(Duration::from_secs(2)) - .crash(Crash::Delay { count: 1, after: 1 }) + .crash(Crash::DelayRound { + participants: vec![delayed], + round: Round::new(Epoch::zero(), View::new(1)), + }) .crash(Crash::Schedule( Schedule::new().at(Duration::from_millis(1), Action::Heal(link)), )) diff --git a/glue/src/simulate/reporter.rs b/glue/src/simulate/reporter.rs index 393c58d28d1..2ad1154de81 100644 --- a/glue/src/simulate/reporter.rs +++ b/glue/src/simulate/reporter.rs @@ -47,7 +47,7 @@ where if let Update::Tip(round, _, ref digest) = activity { let _ = self.monitor.try_send(FinalizationUpdate { pk: self.pk.clone(), - view: round.view(), + round, block_digest: digest.as_ref().to_vec(), }); } diff --git a/glue/src/simulate/tracker.rs b/glue/src/simulate/tracker.rs index 718fc851c33..d7a5fee23b6 100644 --- a/glue/src/simulate/tracker.rs +++ b/glue/src/simulate/tracker.rs @@ -1,6 +1,6 @@ //! Finalization progress tracking and agreement checking. -use commonware_consensus::types::View; +use commonware_consensus::types::{Epoch, Round, View}; use commonware_cryptography::PublicKey; use std::collections::{BTreeMap, HashSet}; @@ -8,8 +8,8 @@ use std::collections::{BTreeMap, HashSet}; pub struct FinalizationUpdate { /// Which validator reported this finalization. pub pk: P, - /// The finalized view. - pub view: View, + /// The finalized round. + pub round: Round, /// The digest of the finalized block (encoded as bytes). pub block_digest: Vec, } @@ -19,18 +19,18 @@ pub struct FinalizationUpdate { /// Validates safety invariants (agreement / no forks) and tracks /// liveness (progress toward a finalization target). pub struct ProgressTracker { - /// Latest finalized view per validator. - status: BTreeMap, + /// Latest finalized round per validator. + status: BTreeMap, - /// Block digests seen at each view (for fork detection). - digests_by_view: BTreeMap>>, + /// Block digests seen at each round (for fork detection). + digests_by_round: BTreeMap>>, } impl Default for ProgressTracker

{ fn default() -> Self { Self { status: BTreeMap::new(), - digests_by_view: BTreeMap::new(), + digests_by_round: BTreeMap::new(), } } } @@ -39,36 +39,36 @@ impl ProgressTracker

{ /// Record a finalization update from a validator. /// /// Returns an error if a different block digest was already seen at - /// the same view (fork detected). + /// the same round (fork detected). /// - /// Strictly lower views are silently ignored: after a crash/restart, + /// Strictly lower rounds are silently ignored: after a crash/restart, /// the consensus engine may replay finalizations a validator has - /// already advanced past. Same-view replays are still checked for + /// already advanced past. Same-round replays are still checked for /// agreement so conflicting digests remain detectable. pub fn observe(&mut self, update: FinalizationUpdate

) -> Result<(), String> { let FinalizationUpdate { pk, - view, + round, block_digest, } = update; - // Skip strictly stale replays after crash/restart. Same-view repeats + // Skip strictly stale replays after crash/restart. Same-round repeats // still go through agreement tracking so conflicting digests remain // detectable. if let Some(prev) = self.status.get(&pk) - && *prev > view + && *prev > round { return Ok(()); } // Check agreement (fork detection) - let digests = self.digests_by_view.entry(view).or_default(); + let digests = self.digests_by_round.entry(round).or_default(); digests.insert(block_digest); if digests.len() > 1 { - return Err(format!("fork detected at view {:?}", view)); + return Err(format!("fork detected at round {:?}", round)); } - self.status.insert(pk, view); + self.status.insert(pk, round); Ok(()) } @@ -77,14 +77,23 @@ impl ProgressTracker

{ let required_view = View::new(required); self.status .values() - .filter(|v| **v >= required_view) + .filter(|round| round.view() >= required_view) .count() >= total } /// Minimum finalized view across all tracked validators. pub fn min_view(&self) -> u64 { - self.status.values().map(|v| v.get()).min().unwrap_or(0) + self.status + .values() + .map(|round| round.view().get()) + .min() + .unwrap_or(0) + } + + /// Highest finalized round observed from any validator. + pub fn max_round(&self) -> Option { + self.status.values().copied().max() } /// Number of validators currently being tracked. @@ -94,8 +103,8 @@ impl ProgressTracker

{ /// Number of unique finalized block digests observed at `view`. pub fn unique_digests_at(&self, view: u64) -> usize { - self.digests_by_view - .get(&View::new(view)) + self.digests_by_round + .get(&Round::new(Epoch::zero(), View::new(view))) .map_or(0, HashSet::len) } } @@ -105,15 +114,30 @@ mod tests { use super::*; use commonware_cryptography::{Signer as _, ed25519}; + fn observe_view( + tracker: &mut ProgressTracker, + seed: u64, + epoch: u64, + view: u64, + ) { + tracker + .observe(FinalizationUpdate { + pk: ed25519::PrivateKey::from_seed(seed).public_key(), + round: Round::new(Epoch::new(epoch), View::new(view)), + block_digest: view.to_le_bytes().to_vec(), + }) + .expect("finalization should be accepted"); + } + #[test] - fn conflicting_same_view_from_same_validator_is_rejected() { + fn conflicting_same_round_from_same_validator_is_rejected() { let pk = ed25519::PrivateKey::from_seed(7).public_key(); let mut tracker = ProgressTracker::default(); tracker .observe(FinalizationUpdate { pk: pk.clone(), - view: View::new(3), + round: Round::new(Epoch::zero(), View::new(3)), block_digest: vec![1, 2, 3], }) .expect("first update should be accepted"); @@ -121,10 +145,10 @@ mod tests { let err = tracker .observe(FinalizationUpdate { pk, - view: View::new(3), + round: Round::new(Epoch::zero(), View::new(3)), block_digest: vec![9, 9, 9], }) - .expect_err("conflicting digest at same view should be rejected"); + .expect_err("conflicting digest at same round should be rejected"); assert!(err.contains("fork detected"), "unexpected error: {err}"); } @@ -137,17 +161,17 @@ mod tests { tracker .observe(FinalizationUpdate { pk: pk1.clone(), - view: View::new(5), + round: Round::new(Epoch::zero(), View::new(5)), block_digest: vec![5, 5, 5], }) .expect("high-watermark update should be accepted"); // A stale replay from pk1 should be ignored and must not influence - // fork detection for that old view. + // fork detection for that old round. tracker .observe(FinalizationUpdate { pk: pk1, - view: View::new(3), + round: Round::new(Epoch::zero(), View::new(3)), block_digest: vec![1, 1, 1], }) .expect("stale replay should be ignored"); @@ -155,9 +179,55 @@ mod tests { tracker .observe(FinalizationUpdate { pk: pk2, - view: View::new(3), + round: Round::new(Epoch::zero(), View::new(3)), block_digest: vec![2, 2, 2], }) .expect("stale replay from another validator should not trigger a fork"); } + + #[test] + fn same_view_in_different_epochs_is_not_a_fork() { + let pk1 = ed25519::PrivateKey::from_seed(1).public_key(); + let pk2 = ed25519::PrivateKey::from_seed(2).public_key(); + let mut tracker = ProgressTracker::default(); + + tracker + .observe(FinalizationUpdate { + pk: pk1, + round: Round::new(Epoch::zero(), View::new(3)), + block_digest: vec![1, 1, 1], + }) + .expect("epoch zero finalization should be accepted"); + + tracker + .observe(FinalizationUpdate { + pk: pk2, + round: Round::new(Epoch::new(1), View::new(3)), + block_digest: vec![2, 2, 2], + }) + .expect("same view in another epoch should not trigger a fork"); + } + + #[test] + fn later_epoch_low_view_does_not_satisfy_required_view() { + let mut tracker = ProgressTracker::default(); + + observe_view(&mut tracker, 1, 1, 1); + observe_view(&mut tracker, 2, 1, 2); + + assert!(!tracker.all_reached(1, 10)); + assert!(!tracker.all_reached(2, 10)); + } + + #[test] + fn exact_required_view_satisfies_progress_threshold() { + let mut tracker = ProgressTracker::default(); + + observe_view(&mut tracker, 1, 0, 10); + observe_view(&mut tracker, 2, 1, 10); + + assert!(tracker.all_reached(1, 10)); + assert!(tracker.all_reached(2, 10)); + assert!(!tracker.all_reached(1, 11)); + } } diff --git a/glue/src/stateful/actor/core/mailbox.rs b/glue/src/stateful/actor/core/mailbox.rs index d62678e95d8..97be33b1c88 100644 --- a/glue/src/stateful/actor/core/mailbox.rs +++ b/glue/src/stateful/actor/core/mailbox.rs @@ -7,19 +7,18 @@ use commonware_actor::{ }; use commonware_consensus::{ Application as ConsensusApplication, CertifiableBlock, Epochable, Reporter, Viewable, - marshal::Update, + marshal::{ + Update, + ancestry::{Ancestry, BoxedAncestry}, + }, }; use commonware_cryptography::Digestible; use commonware_runtime::{Clock, Metrics, Spawner, telemetry::traces::TracedExt as _}; use commonware_utils::{acknowledgement::Exact, channel::oneshot}; -use futures::Stream; use rand_core::Rng; -use std::{collections::VecDeque, pin::Pin, sync::Arc}; +use std::{collections::VecDeque, sync::Arc}; use tracing::{Span, info_span}; -/// Type alias for an ancestor stream sent through the actor mailbox. -pub(crate) type ErasedAncestorStream = Pin> + Send>>; - /// Messages processed by the actor loop. pub(crate) enum Message where @@ -30,7 +29,8 @@ where Propose { span: Span, context: (E, A::Context), - ancestry: ErasedAncestorStream, + ancestry: BoxedAncestry, + parent: A::Input, response: oneshot::Sender>, }, @@ -38,7 +38,7 @@ where Verify { span: Span, context: (E, A::Context), - ancestry: ErasedAncestorStream, + ancestry: BoxedAncestry, response: oneshot::Sender, }, @@ -202,11 +202,13 @@ where type SigningScheme = A::SigningScheme; type Context = A::Context; type Block = A::Block; + type Input = A::Input; async fn propose( &mut self, context: (E, Self::Context), - ancestry: impl Stream> + Send + 'static, + ancestry: impl Ancestry, + parent: Self::Input, ) -> Option { let (response, receiver) = oneshot::channel(); let span = info_span!( @@ -217,7 +219,8 @@ where let _ = self.sender.enqueue(Message::Propose { span, context, - ancestry: Box::pin(ancestry), + ancestry: BoxedAncestry::new(ancestry), + parent, response, }); receiver.await.ok().flatten() @@ -226,7 +229,7 @@ where async fn verify( &mut self, context: (E, Self::Context), - ancestry: impl Stream> + Send + 'static, + ancestry: impl Ancestry, ) -> bool { // We must panic if we don't get a response; We cannot override the decision // of the application based on the availabilitiy of the actor. @@ -239,7 +242,7 @@ where let _ = self.sender.enqueue(Message::Verify { span, context, - ancestry: Box::pin(ancestry), + ancestry: BoxedAncestry::new(ancestry), response, }); receiver diff --git a/glue/src/stateful/actor/core/mod.rs b/glue/src/stateful/actor/core/mod.rs index b3a988e64d9..56bee9a3338 100644 --- a/glue/src/stateful/actor/core/mod.rs +++ b/glue/src/stateful/actor/core/mod.rs @@ -93,8 +93,8 @@ where /// Configuration used to construct the database set. pub db_config: >::Config, - /// Source of input (e.g. transactions) passed to the application on propose. - pub input_provider: A::InputProvider, + /// Provider cloned into each proposal. + pub provider: A::Provider, /// Marshal mailbox used for startup anchoring and lazy recovery. pub marshal: MarshalMailbox, @@ -140,8 +140,8 @@ where /// The inner application that drives state transitions. application: A, - /// Source of input (e.g. transactions) passed to the application on propose. - input_provider: A::InputProvider, + /// Provider cloned into each proposal. + provider: A::Provider, /// Marshal mailbox used for startup anchoring and lazy recovery. marshal: MarshalMailbox, @@ -187,7 +187,7 @@ where context: ContextCell::new(context), mailbox, application: config.application, - input_provider: config.input_provider, + provider: config.provider, marshal: config.marshal, db_config: config.db_config, plan: config.plan, @@ -233,7 +233,7 @@ where context: self.context, mailbox: self.mailbox, application: self.application, - input_provider: self.input_provider, + provider: self.provider, marshal: self.marshal, sync_metadata, syncer: syncer_mailbox, @@ -279,7 +279,7 @@ where Processing { context: self.context, mailbox: self.mailbox, - input_provider: self.input_provider, + provider: self.provider, marshal: self.marshal, processor, skip_finalized_until, @@ -437,7 +437,7 @@ mod tests { Config { application: TestApp, db_config: (), - input_provider: (), + provider: (), marshal, mailbox_size: NZUsize!(8), plan: plan.with_floor(finalization), @@ -458,6 +458,7 @@ mod tests { result = mailbox.propose( (context.child("proposal"), TestBlock::new(1, 1).context()), ancestry::from_iter([]), + (), ) => { assert!(result.is_none()); }, diff --git a/glue/src/stateful/actor/core/processing.rs b/glue/src/stateful/actor/core/processing.rs index 1eee9ccd6a8..5e0005ba0da 100644 --- a/glue/src/stateful/actor/core/processing.rs +++ b/glue/src/stateful/actor/core/processing.rs @@ -1,5 +1,5 @@ use crate::stateful::{ - Application, + Application, Input, actor::{ core::mailbox::Message, processor::{FinalizeStatus, Processor}, @@ -46,8 +46,8 @@ where /// Actor ingress. pub(super) mailbox: actor_mailbox::Receiver>, - /// Source of input (e.g. transactions) passed to the application on propose. - pub(super) input_provider: A::InputProvider, + /// Provider cloned into each proposal. + pub(super) provider: A::Provider, /// Marshal mailbox used for lazy block lookup. pub(super) marshal: MarshalMailbox, @@ -102,16 +102,21 @@ where span, context, ancestry, + parent, response, }) => { let process = info_span!(parent: &span, "stateful.actor.propose"); + let input = Input { + parent, + provider: self.provider.clone(), + }; self.processor .propose( self.context.as_present(), self.marshal.clone(), context, ancestry, - &mut self.input_provider, + input, response, ) .instrument(process) diff --git a/glue/src/stateful/actor/core/syncing.rs b/glue/src/stateful/actor/core/syncing.rs index 1d8581a3f20..c258e1908fa 100644 --- a/glue/src/stateful/actor/core/syncing.rs +++ b/glue/src/stateful/actor/core/syncing.rs @@ -1,10 +1,7 @@ use crate::stateful::{ Application, PruneConfig, actor::{ - core::{ - mailbox::{ErasedAncestorStream, Message}, - processing::Processing, - }, + core::{mailbox::Message, processing::Processing}, metrics::Metrics as StatefulMetrics, processor::{FinalizeStatus, Processor}, syncer::{self, StateSyncMetadata, SyncResult}, @@ -13,9 +10,9 @@ use crate::stateful::{ }; use commonware_actor::mailbox as actor_mailbox; use commonware_consensus::{ - Epochable, Heightable, Viewable, + Block, Epochable, Heightable, Viewable, marshal::{ - ancestry::BlockProvider, + ancestry::{BlockProvider, BoxedAncestry}, core::{Mailbox as MarshalMailbox, Variant}, }, }; @@ -34,10 +31,10 @@ use std::sync::Arc; use tracing::{Instrument as _, Span, debug, error, info_span}; /// Verify request buffered while state sync is still in progress. -pub(super) struct HeldVerify { +pub(super) struct HeldVerify { span: Span, context: C, - ancestry: ErasedAncestorStream, + ancestry: BoxedAncestry, response: oneshot::Sender, } @@ -66,8 +63,8 @@ where /// Inner application. pub(super) application: A, - /// Source of input (e.g. transactions) passed to the application on propose. - pub(super) input_provider: A::InputProvider, + /// Provider cloned into each proposal after state sync. + pub(super) provider: A::Provider, /// Marshal actor mailbox. pub(super) marshal: MarshalMailbox, @@ -313,7 +310,7 @@ where Processing { context: self.context, mailbox: self.mailbox, - input_provider: self.input_provider, + provider: self.provider, marshal: self.marshal, processor, skip_finalized_until: Some(synced_height), @@ -380,7 +377,7 @@ mod tests { context: ContextCell::new(context.child("syncing")), mailbox, application: TestApp, - input_provider: (), + provider: (), marshal: init_marshal_mailbox(context.child("marshal")).await, sync_metadata: Arc::new(AsyncMutex::new( StateSyncMetadata::init(&context, "syncing-test").await, diff --git a/glue/src/stateful/actor/processor/mod.rs b/glue/src/stateful/actor/processor/mod.rs index 9c2c976088f..60348e8d918 100644 --- a/glue/src/stateful/actor/processor/mod.rs +++ b/glue/src/stateful/actor/processor/mod.rs @@ -23,7 +23,7 @@ //! [`await_or_cancel`]. use crate::stateful::{ - Application, Proposed, PruneConfig, + Application, Input, Proposed, PruneConfig, actor::metrics::Metrics as StatefulMetrics, db::{Anchor, DatabaseSet}, }; @@ -31,7 +31,7 @@ use commonware_consensus::{ Block, CertifiableBlock, Heightable, Roundable, marshal::{ Identifier, - ancestry::BlockProvider, + ancestry::{self as marshal_ancestry, Ancestry, BlockProvider}, core::{Mailbox as MarshalMailbox, Variant as MarshalVariant}, }, types::{Height, Round}, @@ -43,7 +43,7 @@ use commonware_runtime::{ telemetry::{metrics::GaugeExt, traces::TracedExt as _}, }; use commonware_utils::channel::{fallible::OneshotExt, oneshot}; -use futures::{Stream, StreamExt, stream}; +use futures::{Stream, StreamExt}; use rand_core::Rng; use std::{ collections::{BTreeMap, HashSet, VecDeque}, @@ -248,8 +248,8 @@ where context: &E, marshal: MarshalMailbox, (runtime_context, consensus_context): (E, A::Context), - ancestry: impl Stream> + Send + 'static, - input_provider: &mut A::InputProvider, + mut ancestry: impl Ancestry, + input: Input, mut response: oneshot::Sender>, ) where S: Scheme, @@ -258,7 +258,6 @@ where { let timer = self.metrics.propose_duration.timer(context); - let mut ancestry = Box::pin(ancestry); let parent = match fetch_ancestor(&mut response, &mut ancestry).await { Some(Some(parent)) => parent, Some(None) => { @@ -271,7 +270,7 @@ where } }; let parent_digest = parent.digest(); - let ancestry = stream::once(std::future::ready(Arc::clone(&parent))).chain(ancestry); + let ancestry = marshal_ancestry::with_prefix([Arc::clone(&parent)], ancestry); let round = consensus_context.round(); let batches = match self @@ -306,7 +305,7 @@ where (runtime_context, consensus_context), ancestry, batches, - input_provider, + input, ), ) .await @@ -340,7 +339,7 @@ where context: &E, marshal: MarshalMailbox, (runtime_context, consensus_context): (E, A::Context), - ancestry: impl Stream> + Send + 'static, + mut ancestry: impl Ancestry, mut response: oneshot::Sender, ) where S: Scheme, @@ -349,7 +348,6 @@ where { let timer = self.metrics.verify_duration.timer(context); - let mut ancestry = Box::pin(ancestry); let block = match fetch_ancestor(&mut response, &mut ancestry).await { Some(Some(block)) => block, Some(None) => { @@ -474,7 +472,7 @@ where } }; - let ancestry = stream::iter([block.clone(), parent]).chain(ancestry); + let ancestry = marshal_ancestry::with_prefix([block.clone(), parent], ancestry); let verified = match await_or_cancel( &mut response, self.app @@ -924,14 +922,14 @@ mod tests { fetch_ancestor, }; use crate::stateful::{ - Application, Proposed, PruneConfig, + Application, Input, Proposed, PruneConfig, actor::metrics::Metrics as StatefulMetrics, db::{Anchor, DatabaseSet, Merkleized as _, Unmerkleized as _}, }; use commonware_codec::{Encode, EncodeSize, Error as CodecError, Read, ReadExt as _, Write}; use commonware_consensus::{ Block as ConsensusBlock, CertifiableBlock, Heightable, Roundable, - marshal::ancestry::BlockProvider, + marshal::ancestry::{Ancestry, BlockProvider}, simplex::{mocks::scheme::Scheme as MockScheme, types::Context as ConsensusContext}, types::{Epoch, Height, Round, View}, }; @@ -955,7 +953,7 @@ mod tests { range::NonEmptyRange, sync::{Mutex, TracedAsyncRwLock}, }; - use futures::{Stream, StreamExt}; + use futures::StreamExt; use std::{ collections::{BTreeMap, VecDeque}, future::Future, @@ -1159,7 +1157,8 @@ mod tests { type Context = TestContext; type Block = Block; type Databases = DbSet; - type InputProvider = (); + type Provider = (); + type Input = (); async fn genesis(&mut self) -> Self::Block { self.genesis.clone() @@ -1168,9 +1167,9 @@ mod tests { async fn propose( &mut self, context: (deterministic::Context, Self::Context), - ancestry: impl Stream> + Send, + ancestry: impl Ancestry, batches: >::Unmerkleized, - _input: &mut Self::InputProvider, + _input: Input, ) -> Option> { let mut ancestry = Box::pin(ancestry); let parent = ancestry.next().await?; @@ -1194,7 +1193,7 @@ mod tests { async fn verify( &mut self, _context: (deterministic::Context, Self::Context), - ancestry: impl Stream> + Send, + ancestry: impl Ancestry, batches: >::Unmerkleized, ) -> Option<>::Merkleized> { let mut ancestry = Box::pin(ancestry); diff --git a/glue/src/stateful/mod.rs b/glue/src/stateful/mod.rs index a0170e9a317..53dfddcc8e0 100644 --- a/glue/src/stateful/mod.rs +++ b/glue/src/stateful/mod.rs @@ -90,13 +90,12 @@ //! [`Inline`]: commonware_consensus::marshal::standard::Inline //! [`coding::Marshaled`]: commonware_consensus::marshal::coding::Marshaled -use commonware_consensus::{CertifiableBlock, Epochable, Viewable}; +use commonware_consensus::{CertifiableBlock, Epochable, Viewable, marshal::ancestry::Ancestry}; use commonware_cryptography::certificate::Scheme; use commonware_runtime::{Clock, Metrics, Spawner}; use db::DatabaseSet; -use futures::Stream; use rand_core::Rng; -use std::{future::Future, sync::Arc}; +use std::future::Future; mod actor; pub use actor::{Config, Mailbox, PruneConfig, Stateful, SyncPlan}; @@ -116,6 +115,23 @@ pub struct Proposed, E: Rng + Spawner + Metrics + Clock> { pub merkleized: >::Merkleized, } +/// Aggregated per-proposal input a [`Stateful`] application hands its inner +/// application. +/// +/// `parent` is the input [`Stateful`] received as a +/// [`commonware_consensus::Application`] (from whatever wraps it); +/// `provider` is the stateful-owned handle from [`Config::provider`]. Being +/// generic over the parent input, it lets an outer application (for example a +/// reshare wrapper) stack its own input on top of the stateful-owned provider +/// without either layer knowing the other. +pub struct Input { + /// Input forwarded from the application wrapping [`Stateful`]. + pub parent: Parent, + + /// Provider owned by the stateful actor, from its [`Config::provider`]. + pub provider: Provider, +} + /// A stateful application whose storage is managed by a [`DatabaseSet`]. /// /// Implementors receive [`DatabaseSet::Unmerkleized`] batches and @@ -148,12 +164,20 @@ where /// The set of databases managed on behalf of this application. type Databases: DatabaseSet; - /// A provider of input to the application. + /// The stateful-owned provider, supplied through + /// [`Config::provider`](crate::stateful::Config::provider). /// /// This may be a mempool that serves transactions, a stream of - /// certificates, or any other source of input that drives state - /// transitions. - type InputProvider: Send; + /// certificates, or any other handle to data that drives state + /// transitions. The stateful actor owns it and clones it for each proposal, + /// so it must be cheap to clone (e.g. `()` or a handle). + type Provider: Send + Clone; + + /// Per-proposal input forwarded from the application wrapping + /// [`Stateful`], aggregated with [`Provider`](Self::Provider) into the + /// [`Input`] handed to [`propose`](Self::propose). Set this to `()` + /// when nothing wraps the stateful actor with its own input. + type Input: Send; /// Extract per-database sync targets from a finalized block. /// @@ -186,9 +210,9 @@ where fn propose( &mut self, context: (E, Self::Context), - ancestry: impl Stream> + Send, + ancestry: impl Ancestry, batches: >::Unmerkleized, - input: &mut Self::InputProvider, + input: Input, ) -> impl Future>> + Send; /// Verify a block received from a peer, relative to its ancestry. @@ -225,7 +249,7 @@ where fn verify( &mut self, context: (E, Self::Context), - ancestry: impl Stream> + Send, + ancestry: impl Ancestry, batches: >::Unmerkleized, ) -> impl Future>::Merkleized>> + Send; diff --git a/glue/src/stateful/probe/actor/discovery.rs b/glue/src/stateful/probe/actor/discovery.rs index 52c480cbd37..427a329858f 100644 --- a/glue/src/stateful/probe/actor/discovery.rs +++ b/glue/src/stateful/probe/actor/discovery.rs @@ -37,7 +37,7 @@ use tracing::debug; pub(super) struct Discovery where E: Spawner + CryptoRng + Clock + Metrics, - S: Scheme, + S: Scheme, D: Provider, V: Variant, T: Strategy, @@ -58,7 +58,7 @@ where impl Discovery where E: Spawner + CryptoRng + Clock + Metrics, - S: Scheme, + S: Scheme, D: Provider, V: Variant, T: Strategy, @@ -112,7 +112,7 @@ where let should_request = self.floor_subscribers.is_empty(); self.floor_subscribers.push(response); if should_request { - Self::request_latest(sender, &mut finalizations); + self.request_latest(sender, &mut finalizations); deadline = self.context.current() + self.retry_timeout.get(); } } @@ -163,7 +163,7 @@ where }, _ = retry => { debug!(reason = "deadline elapsed", "re-requesting finalizations"); - Self::request_latest(sender, &mut finalizations); + self.request_latest(sender, &mut finalizations); deadline = self.context.current() + self.retry_timeout.get(); }, } @@ -207,16 +207,24 @@ where /// Verifies a [`Finalization`] from `peer`. /// - /// Peers that send invalid finalizations are blocked. If no scheme is available for the - /// finalization's epoch, the payload is ignored without blocking because it cannot be judged. + /// Peers outside the solicited participant set or sending invalid finalizations are blocked. + /// If no scheme is available for the finalization's epoch, the payload is ignored without + /// blocking because it cannot be judged. fn verify_finalization( &mut self, peer: P, finalization: Finalization, ) -> Option<(P, Finalization)> { - // Verify against the certificate scheme for the finalization's epoch. If no scheme is + let response_epoch = finalization.epoch(); + let sample_scheme = self.provider.scheme(self.minimum_epoch)?; + if sample_scheme.participants().position(&peer).is_none() { + commonware_p2p::block!(self.blocker, peer, "finalization sent by non-participant"); + return None; + } + + // Verify against the certificate scheme for the finalization's epoch. If no verifier is // available for that epoch, we cannot judge the payload, so ignore it without blocking. - let scoped = self.provider.scoped(finalization.epoch())?; + let scoped = self.provider.scoped(response_epoch)?; if !finalization.verify(self.context.as_present_mut(), &scoped, &self.strategy) { commonware_p2p::block!(self.blocker, peer, "invalid finalization"); return None; @@ -237,7 +245,7 @@ where finalizations .values() .fold((None, 0), |(floor, replies), finalization| { - if self.sample_size(finalization.epoch()).is_none() { + if self.provider.scoped(finalization.epoch()).is_none() { return (floor, replies); } let floor = floor @@ -251,7 +259,7 @@ where let Some(floor) = floor else { return; }; - let Some(sample_size) = self.sample_size(floor.epoch()) else { + let Some(sample_size) = self.sample_size(self.minimum_epoch) else { return; }; if replies < sample_size { @@ -264,14 +272,18 @@ where self.floor = Some(floor.clone()); } - /// Clears any pending responses and re-requests peers' latest [`Finalization`]. + /// Clears any pending responses and requests the current committee's latest [`Finalization`]. fn request_latest( + &self, sender: &mut impl Sender, finalizations: &mut BTreeMap>, ) { finalizations.clear(); + let Some(scheme) = self.provider.scheme(self.minimum_epoch) else { + return; + }; sender.send( - Recipients::All, + Recipients::Some(scheme.participants().iter().cloned().collect()), wire::Message::::Request.encode(), false, ); diff --git a/glue/src/stateful/probe/actor/mod.rs b/glue/src/stateful/probe/actor/mod.rs index a6c5cc15cd8..a70c4203715 100644 --- a/glue/src/stateful/probe/actor/mod.rs +++ b/glue/src/stateful/probe/actor/mod.rs @@ -30,9 +30,10 @@ where pub strategy: T, /// The mailbox capacity. pub capacity: NonZeroUsize, - /// Blocker used to block peers that send invalid finalizations. + /// Blocker used to block malicious peers. pub blocker: B, - /// Finalizations below this epoch are ignored when discovering a floor. + /// Finalizations below this epoch are ignored when discovering a floor. Discovery requests are + /// sent to this epoch's participants. pub minimum_epoch: Epoch, /// How long to wait for enough finalization replies before clearing the pending /// responses and re-requesting. @@ -49,7 +50,7 @@ where pub struct Probe where E: Spawner + CryptoRng + Clock + Metrics, - S: Scheme, + S: Scheme, D: Provider, V: Variant, T: Strategy, @@ -68,7 +69,7 @@ where impl Probe where E: Spawner + CryptoRng + Clock + Metrics, - S: Scheme, + S: Scheme, D: Provider, V: Variant, T: Strategy, diff --git a/glue/src/stateful/probe/mod.rs b/glue/src/stateful/probe/mod.rs index 3040c909753..000d1a0720c 100644 --- a/glue/src/stateful/probe/mod.rs +++ b/glue/src/stateful/probe/mod.rs @@ -9,8 +9,8 @@ //! //! ## Solicit //! -//! Once a floor subscriber appears, [`Probe`] broadcasts a `Request` to every -//! connected peer: +//! Once a floor subscriber appears, [`Probe`] sends a `Request` to each participant in the +//! configured minimum epoch's committee: //! //! ```text //! +-- Request --> peer 1 @@ -29,10 +29,11 @@ //! //! ## Collect and select //! -//! Each peer answers with its own latest finalization (or nothing, if it has none). Every -//! response is verified against the certificate scheme for its epoch. At most one finalization is -//! counted per peer, so no single peer can inflate the sample on its own. Once `f + 1` -//! distinct peers have replied, the highest finalized round becomes the floor: +//! Each peer answers with its own latest finalization (or nothing, if it has none). Every response +//! is verified against the certificate scheme for its epoch, and its sender must be a participant +//! in the committee that was solicited. At most one finalization is counted per peer, so no single +//! peer can inflate the sample on its own. Once `f + 1` distinct peers have replied, the highest +//! finalized round becomes the floor: //! //! ```text //! peer 1 --Response(view 10)-->\ replies @@ -42,9 +43,9 @@ //! sample reached, highest view becomes the floor: 13 //! ``` //! -//! A peer that sends an undecodable or unverifiable first finalization in a request round is -//! blocked. After a peer has already contributed a verified response for that round, later -//! messages from that peer are ignored before validation. +//! A non-participant or a peer that sends an undecodable or unverifiable first finalization in a +//! request round is blocked. After a peer has already contributed a verified response for that +//! round, later messages from that peer are ignored before validation. //! //! ## Retry //! @@ -63,9 +64,10 @@ //! //! # Why the sample is `f + 1` //! -//! The `f` used here comes from the epoch-scoped committee for each accepted response -//! finalization, not from the initially registered peer set. That lets probe tolerate peer churn -//! after startup without lowering the sample below the epoch's fault bound. +//! The `f` used here comes from the configured minimum epoch's committee: the committee that +//! received the request. Returned finalizations are still verified against their own epoch's +//! certificate scheme, so an old-committee member may safely report a newer finalization from a +//! later epoch where it no longer participates. //! //! Assume at most `f` of the `n` participants in that epoch are faulty. In this protocol, `f` makes //! no distinction between Byzantine and crashed nodes: a peer that does not answer and a peer that @@ -88,14 +90,15 @@ //! honest replies. If they report something higher, it must still be a valid finalization, so it is //! a real finalized block rather than a rollback. //! -//! If fewer than `f + 1` registered peers can answer for that epoch, probe cannot resolve that -//! request round and will retry. This is a liveness tradeoff, not a safety one: using the epoch's -//! `f + 1` threshold lets probe tolerate peer churn while preserving the assumption that every -//! completed sample includes at least one honest response from the relevant historical committee. +//! If fewer than `f + 1` solicited peers can answer for that epoch, probe cannot resolve that +//! request round and will retry. This is a liveness tradeoff, not a safety one: using the +//! solicited committee's `f + 1` threshold preserves the assumption that every completed sample +//! includes at least one honest response from the relevant historical committee. //! //! [`Config::minimum_epoch`] bounds that historical search. A caller that initializes peers from a //! known epoch can set it to that lower bound; responses from earlier epochs are ignored. Probe -//! still sizes each accepted sample from that finalization's epoch committee. +//! sizes accepted samples from that lower-bound committee while accepting newer verifiable +//! finalizations reported by its participants. //! //! ```text //! any f + 1 sample: @@ -164,8 +167,8 @@ mod test { }; use commonware_storage::archive::immutable; use commonware_utils::{ - Acknowledgement, NZDuration, NZU16, NZU64, NZUsize, NonZeroDuration, channel::oneshot, - sync::Mutex, test_rng, + Acknowledgement, NZDuration, NZU16, NZU64, NZUsize, NonZeroDuration, TestRng, + channel::oneshot, sync::Mutex, test_rng, }; use std::{ collections::BTreeMap, @@ -1053,6 +1056,89 @@ mod test { }); } + #[test] + fn test_blocks_non_participant_sending_finalization() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|context| async move { + let mut rng = test_rng(); + let Fixture { + participants: committee, + schemes, + .. + } = scheme_mocks::fixture(&mut rng, b"_COMMONWARE_GLUE_PROBE_PARTICIPANTS", 4); + let provider = ConstantProvider::new(schemes[0].clone()); + let mut harness = Harness::setup_with( + &context, + 7, + NZDuration!(Duration::from_secs(3600)), + Epoch::zero(), + move |_scheme| provider.clone(), + ) + .await; + harness.start_probes(); + + let (sender, peer) = harness + .participants + .iter() + .enumerate() + .skip(1) + .find(|(_, peer)| !committee.contains(peer)) + .expect("network should contain a non-participant sender"); + let (_, finalization) = build_finalization(&schemes, 1, 1); + harness.send_raw(sender, 0, finalization_bytes(finalization)); + + context.sleep(Duration::from_millis(100)).await; + + let blocked = harness.oracle.blocked().await.unwrap(); + assert!( + blocked.contains(&(harness.participants[0].clone(), peer.clone())), + "node 0 should have blocked the non-participant sender" + ); + }); + } + + #[test] + fn test_requests_finalizations_only_from_current_participants() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|context| async move { + let mut rng = test_rng(); + let Fixture { + participants: committee, + schemes, + .. + } = scheme_mocks::fixture(&mut rng, b"_COMMONWARE_GLUE_PROBE_PARTICIPANTS", 4); + let provider = ConstantProvider::new(schemes[0].clone()); + let mut harness = Harness::setup_with( + &context, + 7, + NZDuration!(Duration::from_secs(3600)), + Epoch::zero(), + move |_scheme| provider.clone(), + ) + .await; + let (sender, peer) = harness + .participants + .iter() + .enumerate() + .skip(1) + .find(|(_, peer)| !committee.contains(peer)) + .map(|(index, peer)| (index, peer.clone())) + .expect("network should contain a non-participant sender"); + let (block, finalization) = build_finalization(&schemes, 1, 1); + harness.inject(sender, block, finalization).await; + harness.start_probes(); + + let _subscription = harness.nodes[0].probe.subscribe(); + context.sleep(Duration::from_millis(100)).await; + + let blocked = harness.oracle.blocked().await.unwrap(); + assert!( + !blocked.contains(&(harness.participants[0].clone(), peer)), + "a non-participant should not receive a discovery request" + ); + }); + } + #[test] fn test_blocks_peer_sending_invalid_message() { let runner = deterministic::Runner::timed(Duration::from_secs(30)); @@ -1263,12 +1349,12 @@ mod test { fn test_resolves_floor_at_non_zero_epoch() { let runner = deterministic::Runner::timed(Duration::from_secs(30)); runner.start(|context| async move { - // A committee for epoch 1 (f + 1 = 2), distinct from the harness's epoch-0 set. + // A committee for epoch 1, distinct from the solicited epoch-0 set. let mut rng = test_rng(); let Fixture { schemes: epoch_one, .. } = scheme_mocks::fixture(&mut rng, b"_COMMONWARE_GLUE_FD_EPOCH_ONE", 4); - let provider = epoch_provider([(Epoch::new(1), epoch_one[0].clone())]); + let provider = EpochProvider::default(); let mut harness = Harness::setup_with( &context, @@ -1281,6 +1367,8 @@ mod test { }, ) .await; + provider.insert(Epoch::zero(), harness.schemes[0].clone()); + provider.insert(Epoch::new(1), epoch_one[0].clone()); harness.start_probes(); let mut subscription = harness.nodes[0].probe.subscribe(); @@ -1295,6 +1383,72 @@ mod test { }); } + /// A solicited old-committee peer may return a newer finalization from a rotated committee it + /// no longer belongs to. The response is judged against the solicited committee for peer + /// eligibility and sample size, while the certificate is verified against its own epoch. + #[test] + fn test_resolves_rotated_committee_finalization_from_solicited_peers() { + let runner = deterministic::Runner::timed(Duration::from_secs(30)); + runner.start(|context| async move { + let mut rng = TestRng::new(1); + let Fixture { + participants: new_committee, + schemes: new_schemes, + .. + } = scheme_mocks::fixture(&mut rng, b"_COMMONWARE_GLUE_PROBE_ROTATED", 4); + + let provider = EpochProvider::default(); + let mut harness = Harness::setup_with( + &context, + 7, + NZDuration!(Duration::from_secs(3600)), + Epoch::zero(), + { + let provider = provider.clone(); + move |_scheme| provider.clone() + }, + ) + .await; + assert!( + (1..=3).all(|index| !new_committee.contains(&harness.participants[index])), + "test requires old responders to be outside the new committee" + ); + provider.insert(Epoch::zero(), harness.schemes[0].clone()); + provider.insert(Epoch::new(1), new_schemes[0].clone()); + harness.start_probes(); + let mut subscription = harness.nodes[0].probe.subscribe(); + + let (_, finalization) = build_finalization_at(&new_schemes, Epoch::new(1), 1, 7); + for index in 1..=2 { + harness.send_raw(index, 0, finalization_bytes(finalization.clone())); + } + + context.sleep(Duration::from_millis(100)).await; + assert!( + matches!( + subscription.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + ), + "two replies must not satisfy the solicited seven-node committee sample" + ); + let blocked = harness.oracle.blocked().await.unwrap(); + assert!( + blocked.is_empty(), + "old-committee responders must not be blocked for returning a newer finalization" + ); + + harness.send_raw(3, 0, finalization_bytes(finalization.clone())); + context.sleep(Duration::from_millis(100)).await; + let floor = subscription.try_recv().expect("floor resolved"); + assert_eq!(floor, finalization); + let blocked = harness.oracle.blocked().await.unwrap(); + assert!( + blocked.is_empty(), + "no solicited responder should be blocked" + ); + }); + } + /// Finalizations below the configured minimum epoch are ignored without blocking and do not /// prevent the same peers from contributing accepted finalizations later in the round. #[test] @@ -1512,6 +1666,7 @@ mod test { }, ) .await; + provider.insert(Epoch::zero(), harness.schemes[0].clone()); provider.insert(Epoch::new(1), harness.schemes[0].clone()); provider.insert(Epoch::new(2), harness.schemes[0].clone()); harness.start_probes(); diff --git a/glue/src/stateful/tests/mocks.rs b/glue/src/stateful/tests/mocks.rs index 319dc16ef14..50cad91153e 100644 --- a/glue/src/stateful/tests/mocks.rs +++ b/glue/src/stateful/tests/mocks.rs @@ -1,11 +1,11 @@ use crate::stateful::{ - Application, Proposed, + Application, Input, Proposed, db::{DatabaseSet, ManagedDb, Merkleized, Unmerkleized}, }; use commonware_codec::{EncodeSize, Error as CodecError, Read, ReadExt as _, Write}; use commonware_consensus::{ Block as ConsensusBlock, CertifiableBlock, Heightable, - marshal::standard::Standard, + marshal::{ancestry::Ancestry, standard::Standard}, simplex::{mocks::scheme as scheme_mocks, types::Context as SimplexContext}, types::{Epoch, Height, View}, }; @@ -14,7 +14,6 @@ use commonware_cryptography::{ }; use commonware_runtime::{Buf, BufMut, deterministic}; use commonware_utils::sync::TracedAsyncRwLock; -use futures::Stream; use std::{convert::Infallible, sync::Arc}; pub(crate) type TestDatabases = Arc>; @@ -175,7 +174,8 @@ impl Application for TestApp { type Context = SimplexContext; type Block = TestBlock; type Databases = TestDatabases; - type InputProvider = (); + type Provider = (); + type Input = (); fn sync_targets( block: &Self::Block, @@ -190,9 +190,9 @@ impl Application for TestApp { async fn propose( &mut self, _context: (deterministic::Context, Self::Context), - _ancestry: impl Stream> + Send, + _ancestry: impl Ancestry, _batches: >::Unmerkleized, - _input: &mut Self::InputProvider, + _input: Input, ) -> Option> { None } @@ -200,7 +200,7 @@ impl Application for TestApp { async fn verify( &mut self, _context: (deterministic::Context, Self::Context), - _ancestry: impl Stream> + Send, + _ancestry: impl Ancestry, _batches: >::Unmerkleized, ) -> Option<>::Merkleized> { None diff --git a/glue/src/stateful/tests/mod.rs b/glue/src/stateful/tests/mod.rs index 02fb668f79d..79f204be9a5 100644 --- a/glue/src/stateful/tests/mod.rs +++ b/glue/src/stateful/tests/mod.rs @@ -8,6 +8,7 @@ use crate::simulate::{ processed::ProcessedHeight, property::Property, }; +use commonware_consensus::types::{Epoch, Round, View}; use commonware_cryptography::{PublicKey, ed25519}; use commonware_macros::{test_group, test_traced}; use commonware_p2p::simulated::Link; @@ -28,6 +29,13 @@ mod single_db_app; const NUM_VALIDATORS: u32 = 5; +fn delay_first(participants: &[P], view: u64) -> Crash

{ + Crash::DelayRound { + participants: vec![participants[0].clone()], + round: Round::new(Epoch::zero(), View::new(view)), + } +} + #[test_group("slow")] #[test_traced("DEBUG")] fn all_validators_finalize_and_commit() { @@ -368,9 +376,10 @@ where BlockAgreementAtHeight: Property, ProcessedHeightAtLeast: ExitCondition, { + let delay = delay_first(&engine.participants(), 5); PlanBuilder::new(engine) .seeds(0..5) - .crash(Crash::Delay { count: 1, after: 5 }) + .crash(delay) .exit_condition(ProcessedHeightAtLeast::new(20)) .property(BlockAgreementAtHeight::new(20)) .run() @@ -414,12 +423,10 @@ where LateJoinerStateSyncHandoff: Property, ProcessedHeightAtLeast: ExitCondition, { + let delay = delay_first(&engine.participants(), 80); PlanBuilder::new(engine) .seeds(0..5) - .crash(Crash::Delay { - count: 1, - after: 80, - }) + .crash(delay) .exit_condition(ProcessedHeightAtLeast::new(150)) .property(LateJoinerStateSyncHandoff) .property(BlockAgreementAtHeight::new(150)) @@ -562,12 +569,10 @@ where ProcessedHeightAtLeast: ExitCondition, { let seeds = 0..5; + let delay = delay_first(&engine.participants(), 80); let r1 = PlanBuilder::new(engine.clone()) .seeds(seeds.clone()) - .crash(Crash::Delay { - count: 1, - after: 80, - }) + .crash(delay.clone()) .exit_condition(ProcessedHeightAtLeast::new(100)) .property(LateJoinerStateSyncHandoff) .property(BlockAgreementAtHeight::new(100)) @@ -575,10 +580,7 @@ where .unwrap(); let r2 = PlanBuilder::new(engine) .seeds(seeds.clone()) - .crash(Crash::Delay { - count: 1, - after: 80, - }) + .crash(delay) .exit_condition(ProcessedHeightAtLeast::new(100)) .property(LateJoinerStateSyncHandoff) .property(BlockAgreementAtHeight::new(100)) @@ -600,12 +602,10 @@ where LateJoinerStateSyncHandoff: Property, ProcessedHeightAtLeast: ExitCondition, { + let delay = delay_first(&engine.participants(), 80); PlanBuilder::new(engine) .seeds(0..5) - .crash(Crash::Delay { - count: 1, - after: 80, - }) + .crash(delay) .crash(Crash::Random { frequency: Duration::from_secs(3), downtime: Duration::from_secs(1), @@ -626,12 +626,10 @@ where LateJoinerStateSyncHandoff: Property, ProcessedHeightAtLeast: ExitCondition, { + let delay = delay_first(&engine.participants(), 30); PlanBuilder::new(engine) .seeds(0..5) - .crash(Crash::Delay { - count: 1, - after: 30, - }) + .crash(delay) .link(link) .exit_condition(ProcessedHeightAtLeast::new(60)) .property(LateJoinerStateSyncHandoff) @@ -654,9 +652,9 @@ where let late_joiner = engine.participants()[0].clone(); PlanBuilder::new(engine) .seeds(0..5) - .crash(Crash::Delay { - count: 1, - after: 80, + .crash(Crash::DelayRound { + participants: vec![late_joiner.clone()], + round: Round::new(Epoch::zero(), View::new(80)), }) // Crash the late joiner while it is still catching up through startup // state sync, then restart it without clearing any partitions. @@ -688,9 +686,9 @@ where let late_joiner = participants[0].clone(); PlanBuilder::new(engine) .seeds(0..5) - .crash(Crash::Delay { - count: 1, - after: 20, + .crash(Crash::DelayRound { + participants: vec![late_joiner.clone()], + round: Round::new(Epoch::zero(), View::new(20)), }) .crash(Crash::Schedule(state_sync_partitioned_restart_schedule( &participants, diff --git a/glue/src/stateful/tests/multi_db_app.rs b/glue/src/stateful/tests/multi_db_app.rs index b6963c69f87..831171b9aea 100644 --- a/glue/src/stateful/tests/multi_db_app.rs +++ b/glue/src/stateful/tests/multi_db_app.rs @@ -5,8 +5,8 @@ use crate::{ reporter::MonitorReporter, }, stateful::{ - Application, Config as StatefulConfig, Proposed, PruneConfig, Stateful as StatefulActor, - SyncPlan, + Application, Config as StatefulConfig, Input, Proposed, PruneConfig, + Stateful as StatefulActor, SyncPlan, db::{ DatabaseSet, Merkleized as _, SyncEngineConfig, Unmerkleized as _, p2p::{compact as compact_resolver, standard as qmdb_resolver}, @@ -20,6 +20,7 @@ use commonware_consensus::{ Block as ConsensusBlock, CertifiableBlock, Heightable, marshal::{ self, + ancestry::Ancestry, core::{Actor as MarshalActor, CommitmentFallback}, resolver::p2p as marshal_resolver, standard::{Deferred, Standard}, @@ -61,7 +62,7 @@ use commonware_utils::{ sync::{Mutex, TracedAsyncRwLock}, test_rng, }; -use futures::{Stream, StreamExt}; +use futures::StreamExt; use rand_core::Rng; use std::{collections::BTreeMap, sync::Arc, time::Duration}; @@ -242,7 +243,8 @@ impl Application for App { type Context = Context; type Block = Block; type Databases = MultiDatabaseSet; - type InputProvider = (); + type Provider = (); + type Input = (); async fn genesis(&mut self) -> Self::Block { self.genesis.clone() @@ -251,9 +253,9 @@ impl Application for App { async fn propose( &mut self, context: (E, Self::Context), - ancestry: impl Stream> + Send, + ancestry: impl Ancestry, batches: >::Unmerkleized, - _input: &mut Self::InputProvider, + _input: Input, ) -> Option> { let mut ancestry = Box::pin(ancestry); let parent = ancestry.next().await?; @@ -285,7 +287,7 @@ impl Application for App { async fn verify( &mut self, _context: (E, Self::Context), - ancestry: impl Stream> + Send, + ancestry: impl Ancestry, batches: >::Unmerkleized, ) -> Option<>::Merkleized> { let mut ancestry = Box::pin(ancestry); @@ -635,7 +637,7 @@ impl EngineDefinition for MultiDbEngine { StatefulConfig { application, db_config, - input_provider: (), + provider: (), marshal: marshal_mailbox.clone(), mailbox_size: NZUsize!(100), plan, diff --git a/glue/src/stateful/tests/single_db_app.rs b/glue/src/stateful/tests/single_db_app.rs index 6eb49923a50..c4b5cd3d971 100644 --- a/glue/src/stateful/tests/single_db_app.rs +++ b/glue/src/stateful/tests/single_db_app.rs @@ -5,8 +5,8 @@ use crate::{ reporter::MonitorReporter, }, stateful::{ - Application, Config as StatefulConfig, Proposed, PruneConfig, Stateful as StatefulActor, - SyncPlan, + Application, Config as StatefulConfig, Input, Proposed, PruneConfig, + Stateful as StatefulActor, SyncPlan, db::{ DatabaseSet, Merkleized as _, SyncEngineConfig, Unmerkleized as _, p2p::standard as qmdb_resolver, @@ -20,6 +20,7 @@ use commonware_consensus::{ Block as ConsensusBlock, CertifiableBlock, Heightable, marshal::{ self, + ancestry::Ancestry, core::{Actor as MarshalActor, CommitmentFallback}, resolver::p2p as marshal_resolver, standard::{Deferred, Standard}, @@ -59,7 +60,7 @@ use commonware_utils::{ sync::{Mutex, TracedAsyncRwLock}, test_rng, }; -use futures::{Stream, StreamExt}; +use futures::StreamExt; use rand_core::Rng; use std::{collections::BTreeMap, sync::Arc, time::Duration}; @@ -193,7 +194,8 @@ impl Application for App { type Context = Context; type Block = Block; type Databases = SingleDatabaseSet; - type InputProvider = (); + type Provider = (); + type Input = (); async fn genesis(&mut self) -> Self::Block { self.genesis.clone() @@ -202,9 +204,9 @@ impl Application for App { async fn propose( &mut self, context: (E, Self::Context), - ancestry: impl Stream> + Send, + ancestry: impl Ancestry, batches: >::Unmerkleized, - _input: &mut Self::InputProvider, + _input: Input, ) -> Option> { let mut ancestry = Box::pin(ancestry); let parent = ancestry.next().await?; @@ -224,7 +226,7 @@ impl Application for App { async fn verify( &mut self, _context: (E, Self::Context), - ancestry: impl Stream> + Send, + ancestry: impl Ancestry, batches: >::Unmerkleized, ) -> Option<>::Merkleized> { let mut ancestry = Box::pin(ancestry); @@ -505,7 +507,7 @@ impl EngineDefinition for SingleDbEngine { StatefulConfig { application, db_config, - input_provider: (), + provider: (), marshal: marshal_mailbox.clone(), mailbox_size: NZUsize!(100), plan, diff --git a/macros/impl/src/lib.rs b/macros/impl/src/lib.rs index 9d8c9d32d09..30e1333e0a2 100644 --- a/macros/impl/src/lib.rs +++ b/macros/impl/src/lib.rs @@ -321,7 +321,6 @@ pub fn test_traced(attr: TokenStream, item: TokenStream) -> TokenStream { tracing_subscriber::fmt::layer() .with_test_writer() .with_line_number(true) - .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE) ) .with(filter); let dispatcher = tracing::Dispatch::new(subscriber);