diff --git a/pepper-sync/src/sync.rs b/pepper-sync/src/sync.rs index 7afb39619e..8a54098a92 100644 --- a/pepper-sync/src/sync.rs +++ b/pepper-sync/src/sync.rs @@ -43,7 +43,7 @@ use crate::wallet::{ KeyIdInterface, NoteInterface, NullifierMap, OutputId, OutputInterface, PoolActivation, ScanTarget, SyncMode, SyncState, WalletBlock, WalletTransaction, }; -use crate::witness::LocatedTreeData; +use crate::witness::{ANCHOR_RETENTION_INTERVALS, LocatedTreeData}; use crate::witness; @@ -62,6 +62,13 @@ pub mod truncate; /// its boundary, this constant is the one place that follows it. pub const MAX_REORG_ALLOWANCE: u32 = 100; +/// The maximum number of checkpoints in the rolling window for re-org handling and chain tip anchor spends. +pub const SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE: u32 = MAX_REORG_ALLOWANCE + 1; + +/// The maximum total number of checkpoints a shard tree persists. +pub const MAX_SHARDTREE_CHECKPOINTS: u32 = + SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE + ANCHOR_RETENTION_INTERVALS; + const VERIFY_BLOCK_RANGE_SIZE: u32 = 10; /// A snapshot of the current state of sync. Useful for displaying the status of sync to a user / consumer. @@ -446,6 +453,8 @@ where consensus_parameters, )?; + repin_anchor_checkpoints(consensus_parameters, &mut *wallet.write().await)?; + let ufvks = wallet .read() .await @@ -1726,8 +1735,9 @@ where H: incrementalmerkletree::Hashable + Clone + PartialEq, { let mut truncation_height = None; + let checkpoint_count = tree.store().checkpoint_count().expect("infallible"); tree.store() - .for_each_checkpoint((MAX_REORG_ALLOWANCE + 1) as usize, |height, _| { + .for_each_checkpoint(checkpoint_count, |height, _| { if truncation_height.is_some() { return Ok(()); } @@ -2030,6 +2040,7 @@ where fetch_request_sender, scan_range, highest_scanned_height, + witness::anchor_retention_policy(consensus_parameters), sapling_located_trees, orchard_located_trees, ironwood_located_trees, @@ -2309,6 +2320,36 @@ where Ok(()) } +/// Re-derives the pinned anchor-checkpoint set of each shard tree from the retention policy in +/// force for `consensus_parameters`, as of the wallet's newest scanned block. +fn repin_anchor_checkpoints( + consensus_parameters: &impl consensus::Parameters, + wallet: &mut W, +) -> Result<(), SyncError> +where + W: SyncWallet + SyncShardTrees, +{ + let Some(policy) = witness::anchor_retention_policy(consensus_parameters) else { + return Ok(()); + }; + let Some(highest_scanned_height) = wallet + .get_sync_state() + .map_err(SyncError::WalletError)? + .highest_scanned_height() + else { + return Ok(()); + }; + let window = witness::anchor_retention_window(&policy, highest_scanned_height); + let shard_trees = wallet + .get_shard_trees_mut() + .map_err(SyncError::WalletError)?; + witness::repin_anchor_checkpoints(&policy, &window, shard_trees.sapling.store_mut()); + witness::repin_anchor_checkpoints(&policy, &window, shard_trees.orchard.store_mut()); + witness::repin_anchor_checkpoints(&policy, &window, shard_trees.ironwood.store_mut()); + + Ok(()) +} + async fn add_initial_frontier( consensus_parameters: &impl consensus::Parameters, fetch_request_sender: mpsc::UnboundedSender, diff --git a/pepper-sync/src/wallet.rs b/pepper-sync/src/wallet.rs index b82016ad9d..6949e6d6d6 100644 --- a/pepper-sync/src/wallet.rs +++ b/pepper-sync/src/wallet.rs @@ -40,7 +40,7 @@ use crate::{ keys::{self, KeyId, transparent::TransparentAddressId}, scan::compact_blocks::calculate_block_tree_bounds, shardtree_ext::{CheckpointAppendOutcome, ShardTreeExt as _}, - sync::{MAX_REORG_ALLOWANCE, ScanPriority, ScanRange}, + sync::{SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE, ScanPriority, ScanRange}, utils::{block, transaction}, witness, }; @@ -1828,7 +1828,10 @@ pub(crate) fn empty_shard_tree() where H: incrementalmerkletree::Hashable + Clone + PartialEq, { - let mut tree = ShardTree::new(MemoryShardStore::empty(), MAX_REORG_ALLOWANCE as usize); + let mut tree = ShardTree::new( + MemoryShardStore::empty(), + SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE as usize, + ); // `NotAboveNewest` is impossible on an empty checkpoint store. assert_eq!( diff --git a/pepper-sync/src/wallet/serialization.rs b/pepper-sync/src/wallet/serialization.rs index 8327c21ebf..58735455fa 100644 --- a/pepper-sync/src/wallet/serialization.rs +++ b/pepper-sync/src/wallet/serialization.rs @@ -35,7 +35,7 @@ use crate::{ KeyId, decode_unified_address, transparent::{TransparentAddressId, TransparentScope}, }, - sync::{MAX_REORG_ALLOWANCE, ScanPriority, ScanRange}, + sync::{SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE, ScanPriority, ScanRange}, wallet::ScanTarget, }; @@ -1242,7 +1242,7 @@ impl ShardTrees { Ok(shardtree::ShardTree::new( store, - MAX_REORG_ALLOWANCE as usize, + SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE as usize, )) } @@ -1323,7 +1323,10 @@ impl ShardTrees { macro_rules! write_with_error_handling { ($writer: ident, $from: ident) => { if let Err(e) = $writer(&mut writer, &$from) { - *shardtree = shardtree::ShardTree::new(store, MAX_REORG_ALLOWANCE as usize); + *shardtree = shardtree::ShardTree::new( + store, + SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE as usize, + ); return Err(e); } }; @@ -1341,17 +1344,14 @@ impl ShardTrees { Ok(()) }) .expect("Infallible"); - if checkpoints.len() > MAX_REORG_ALLOWANCE as usize { - let keep_from = checkpoints.len() - MAX_REORG_ALLOWANCE as usize; - checkpoints.drain(..keep_from); - } write_with_error_handling!(write_checkpoints, checkpoints); // Write cap let cap = store.get_cap().expect("Infallible"); write_with_error_handling!(write_shard, cap); - *shardtree = shardtree::ShardTree::new(store, MAX_REORG_ALLOWANCE as usize); + *shardtree = + shardtree::ShardTree::new(store, SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE as usize); Ok(()) } @@ -1359,6 +1359,8 @@ impl ShardTrees { #[cfg(test)] mod tests { + use crate::{sync::MAX_SHARDTREE_CHECKPOINTS, witness::ANCHOR_RETENTION_INTERVALS}; + use super::*; // Helper: build a minimal v3 SyncState byte blob (no ironwood_shard_ranges). @@ -1467,27 +1469,124 @@ mod tests { assert_eq!(recovered.ironwood_final_tree_size, 6); } + /// The checkpoint set of a synced wallet decomposes into exactly two parts: + /// [`SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE`] rolling checkpoints, which serve ordinary reorg handling and + /// near-tip spends, plus [`ANCHOR_RETENTION_INTERVALS`] pinned ZIP 318 grid boundaries, + /// which serve pool crossings. + /// + /// The two parts are disjoint and independently bounded: pinning a boundary must not + /// consume a rolling slot (that would shrink the reorg window), and the rolling budget must + /// not displace a boundary (that would break crossings). + #[test] + fn checkpoint_set_is_reorg_window_plus_pinned_boundaries() { + use crate::shardtree_ext::ShardTreeExt as _; + use crate::witness::{anchor_retention_window, repin_anchor_checkpoints}; + use zcash_client_backend::data_api::anchor_retention::{ + AnchorRetention, AnchorRetentionInterval, + }; + + const TIP: u32 = 100_000; + const INTERVAL: u32 = 144; + let policy = AnchorRetention::new( + BlockHeight::from_u32(90_000), + AnchorRetentionInterval::default(), + ); + let mut shard_trees = ShardTrees::new(); + + for height in (TIP - 2000)..=TIP { + let height = BlockHeight::from_u32(height); + let window = anchor_retention_window(&policy, height); + repin_anchor_checkpoints(&policy, &window, shard_trees.orchard.store_mut()); + shard_trees + .orchard + .append_checkpoint(height) + .expect("infallible"); + } + + let store = shard_trees.orchard.store(); + let total = store.checkpoint_count().expect("infallible"); + let pinned_ids = store.retained_checkpoints().expect("infallible"); + let mut pinned = Vec::new(); + let mut rolling = Vec::new(); + store + .for_each_checkpoint(total, |id, _| { + if pinned_ids.contains(id) { + pinned.push(u32::from(*id)); + } else { + rolling.push(u32::from(*id)); + } + Ok(()) + }) + .expect("infallible"); + + assert!( + pinned.iter().all(|height| height % INTERVAL == 0), + "every pinned checkpoint must be a grid boundary, got {pinned:?}" + ); + assert_eq!(rolling.last().copied(), Some(TIP)); + assert_eq!( + (rolling.len(), pinned.len(), total), + ( + SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE as usize, + ANCHOR_RETENTION_INTERVALS as usize, + MAX_SHARDTREE_CHECKPOINTS as usize, + ), + "(rolling, pinned, total): the pinned boundaries must not be part of the \ + SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE total" + ); + + for height in (TIP - SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE + 1)..=TIP { + assert!( + store + .get_checkpoint(&BlockHeight::from_u32(height)) + .expect("infallible") + .is_some(), + "reorg window is missing height {height}" + ); + } + + assert!( + rolling + .iter() + .all(|height| *height >= TIP - SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE), + "a rolling checkpoint survived below the reorg window: {rolling:?}" + ); + + let mut bytes = Vec::new(); + shard_trees.write(&mut bytes).expect("write should succeed"); + let reloaded = ShardTrees::read(bytes.as_slice()).expect("read should succeed"); + let reloaded_store = reloaded.orchard.store(); + for boundary in &pinned { + assert!( + reloaded_store + .get_checkpoint(&BlockHeight::from_u32(*boundary)) + .expect("infallible") + .is_some(), + "boundary {boundary} lost on reload; a crossing anchored there cannot be built" + ); + } + assert_eq!( + reloaded_store.checkpoint_count().expect("infallible"), + MAX_SHARDTREE_CHECKPOINTS as usize + ); + } + + /// Serialization preserves the checkpoints pruning left instead of imposing a cap. #[test] fn shardtree_roundtrip_keeps_newest_checkpoints() { + use crate::shardtree_ext::ShardTreeExt as _; + let mut shard_trees = ShardTrees::new(); for height in 1..=150 { let height = BlockHeight::from_u32(height); shard_trees .sapling - .store_mut() - .add_checkpoint( - height, - Checkpoint::from_parts(TreeState::Empty, BTreeSet::new()), - ) + .append_checkpoint(height) .expect("infallible"); shard_trees .orchard - .store_mut() - .add_checkpoint( - height, - Checkpoint::from_parts(TreeState::Empty, BTreeSet::new()), - ) + .append_checkpoint(height) .expect("infallible"); } @@ -1498,35 +1597,87 @@ mod tests { let sapling_store = roundtripped.sapling.store(); let orchard_store = roundtripped.orchard.store(); - assert_eq!(sapling_store.checkpoint_count().expect("infallible"), 100); - assert_eq!(orchard_store.checkpoint_count().expect("infallible"), 100); - assert_eq!( - sapling_store.min_checkpoint_id().expect("infallible"), - Some(BlockHeight::from_u32(51)) - ); - assert_eq!( - sapling_store.max_checkpoint_id().expect("infallible"), - Some(BlockHeight::from_u32(150)) - ); - assert_eq!( - orchard_store.min_checkpoint_id().expect("infallible"), - Some(BlockHeight::from_u32(51)) - ); - assert_eq!( - orchard_store.max_checkpoint_id().expect("infallible"), - Some(BlockHeight::from_u32(150)) + let oldest_kept = BlockHeight::from_u32(150 - SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE + 1); + fn assert_window(store: &S, oldest_kept: BlockHeight) + where + S: ShardStore, + { + assert_eq!( + store.checkpoint_count().expect("infallible"), + SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE as usize + ); + assert_eq!( + store.min_checkpoint_id().expect("infallible"), + Some(oldest_kept) + ); + assert_eq!( + store.max_checkpoint_id().expect("infallible"), + Some(BlockHeight::from_u32(150)) + ); + assert!( + store + .get_checkpoint(&(oldest_kept - 1)) + .expect("infallible") + .is_none() + ); + } + assert_window(sapling_store, oldest_kept); + assert_window(orchard_store, oldest_kept); + } + + /// A pinned anchor checkpoint survives serialization even once it has aged out of the + /// rolling window. The pinned set itself is not persisted, being re-derived at the start of + /// every sync. + #[test] + fn shardtree_roundtrip_keeps_pinned_anchor_checkpoints() { + use crate::shardtree_ext::ShardTreeExt as _; + + let pinned = BlockHeight::from_u32(24); + let mut shard_trees = ShardTrees::new(); + + shard_trees + .sapling + .store_mut() + .add_retained_checkpoint(pinned) + .expect("infallible"); + for height in 1..=150 { + shard_trees + .sapling + .append_checkpoint(BlockHeight::from_u32(height)) + .expect("infallible"); + } + + assert!( + shard_trees + .sapling + .store() + .get_checkpoint(&pinned) + .expect("infallible") + .is_some(), + "pruning must not evict a pinned checkpoint" ); + + let mut bytes = Vec::new(); + shard_trees.write(&mut bytes).expect("write should succeed"); + let roundtripped = ShardTrees::read(bytes.as_slice()).expect("read should succeed"); + + let sapling_store = roundtripped.sapling.store(); assert!( sapling_store - .get_checkpoint(&BlockHeight::from_u32(149)) + .get_checkpoint(&pinned) .expect("infallible") - .is_some() + .is_some(), + "serialization must not evict a pinned checkpoint" + ); + assert_eq!( + sapling_store.checkpoint_count().expect("infallible"), + SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE as usize + 1 ); assert!( sapling_store - .get_checkpoint(&BlockHeight::from_u32(50)) + .retained_checkpoints() .expect("infallible") - .is_none() + .is_empty() ); } } diff --git a/pepper-sync/src/wallet/traits.rs b/pepper-sync/src/wallet/traits.rs index 9ab0d866ba..0114423dd0 100644 --- a/pepper-sync/src/wallet/traits.rs +++ b/pepper-sync/src/wallet/traits.rs @@ -10,6 +10,7 @@ use orchard::tree::MerkleHashOrchard; use shardtree::ShardTree; use shardtree::store::memory::MemoryShardStore; use shardtree::store::{Checkpoint, ShardStore, TreeState}; +use zcash_client_backend::data_api::anchor_retention::AnchorRetention; use zcash_keys::keys::UnifiedFullViewingKey; use zcash_primitives::transaction::TxId; use zcash_protocol::consensus::BlockHeight; @@ -21,7 +22,7 @@ use crate::keys::transparent::TransparentAddressId; use crate::reset_spends; use crate::shardtree_ext::{RollbackOutcome, ShardTreeExt}; use crate::sync::truncate::{PoolTruncation, plan_pool_truncation, tree_facts}; -use crate::sync::{MAX_REORG_ALLOWANCE, ScanRange}; +use crate::sync::{MAX_SHARDTREE_CHECKPOINTS, ScanRange}; use crate::wallet::{ Ironwood, NullifierMap, Orchard, OutputId, Sapling, ShardTrees, SyncState, WalletBlock, WalletTransaction, empty_shard_tree, @@ -252,11 +253,13 @@ pub trait SyncShardTrees: SyncWallet { /// Update wallet shard trees with new shard tree data. /// /// `highest_scanned_height` is the height of the highest scanned block in the wallet not including the `scan_range` we are updating. + #[allow(clippy::too_many_arguments)] fn update_shard_trees( &mut self, fetch_request_sender: mpsc::UnboundedSender, scan_range: &ScanRange, highest_scanned_height: BlockHeight, + anchor_retention: Option, sapling_located_trees: Vec>, orchard_located_trees: Vec>, ironwood_located_trees: Vec>, @@ -267,22 +270,22 @@ pub trait SyncShardTrees: SyncWallet { async move { let shard_trees = self.get_shard_trees_mut().map_err(SyncError::WalletError)?; - // limit the range that checkpoints are manually added to the top MAX_REORG_ALLOWANCE scanned blocks for efficiency. + // limit the range that checkpoints are manually added to the top MAX_SHARDTREE_CHECKPOINTS scanned blocks for efficiency. // As we sync the chain tip first and have spend-before-sync, we will always choose anchors very close to chain // height and we will also never need to truncate to checkpoints lower than this height. let checkpoint_range = if scan_range.block_range().start > highest_scanned_height { let verification_window_start = scan_range .block_range() .end - .saturating_sub(MAX_REORG_ALLOWANCE); + .saturating_sub(MAX_SHARDTREE_CHECKPOINTS); std::cmp::max(scan_range.block_range().start, verification_window_start) ..scan_range.block_range().end } else if scan_range.block_range().end - > highest_scanned_height.saturating_sub(MAX_REORG_ALLOWANCE) + 1 + > highest_scanned_height.saturating_sub(MAX_SHARDTREE_CHECKPOINTS) + 1 { let verification_window_start = - highest_scanned_height.saturating_sub(MAX_REORG_ALLOWANCE) + 1; + highest_scanned_height.saturating_sub(MAX_SHARDTREE_CHECKPOINTS) + 1; std::cmp::max(scan_range.block_range().start, verification_window_start) ..scan_range.block_range().end @@ -336,6 +339,71 @@ pub trait SyncShardTrees: SyncWallet { .await?; } + if let Some(retention) = &anchor_retention { + let as_of = std::cmp::max(highest_scanned_height, scan_range.block_range().end - 1); + let window = witness::anchor_retention_window(retention, as_of); + witness::repin_anchor_checkpoints( + retention, + &window, + shard_trees.sapling.store_mut(), + ); + witness::repin_anchor_checkpoints( + retention, + &window, + shard_trees.orchard.store_mut(), + ); + witness::repin_anchor_checkpoints( + retention, + &window, + shard_trees.ironwood.store_mut(), + ); + + let start = std::cmp::max(*window.start(), scan_range.block_range().start); + let end = std::cmp::min(*window.end(), scan_range.block_range().end - 1); + for boundary in retention.retained_in_range(start..=end) { + if checkpoint_range.contains(&boundary) { + continue; + } + add_checkpoint::< + Sapling, + sapling_crypto::Node, + { sapling_crypto::NOTE_COMMITMENT_TREE_DEPTH }, + { witness::SHARD_HEIGHT }, + >( + fetch_request_sender.clone(), + boundary, + &sapling_located_trees, + &mut shard_trees.sapling, + ) + .await?; + add_checkpoint::< + Orchard, + MerkleHashOrchard, + { orchard::NOTE_COMMITMENT_TREE_DEPTH as u8 }, + { witness::SHARD_HEIGHT }, + >( + fetch_request_sender.clone(), + boundary, + &orchard_located_trees, + &mut shard_trees.orchard, + ) + .await?; + add_checkpoint::< + Ironwood, + MerkleHashOrchard, + { orchard::NOTE_COMMITMENT_TREE_DEPTH as u8 }, + { witness::SHARD_HEIGHT }, + >( + fetch_request_sender.clone(), + boundary, + &ironwood_located_trees, + &mut shard_trees.ironwood, + ) + .await?; + } + } + + // TODO: use `batch_insert_trees` for tree in sapling_located_trees { shard_trees .sapling @@ -461,12 +529,15 @@ where let mut previous_checkpoint = None; shard_tree .store() - .for_each_checkpoint(1_000, |height, checkpoint| { - if *height == checkpoint_height - 1 { - previous_checkpoint = Some(checkpoint.clone()); - } - Ok(()) - }) + .for_each_checkpoint( + MAX_SHARDTREE_CHECKPOINTS as usize + 100, + |height, checkpoint| { + if *height == checkpoint_height - 1 { + previous_checkpoint = Some(checkpoint.clone()); + } + Ok(()) + }, + ) .expect("infallible"); let tree_state = if let Some(checkpoint) = previous_checkpoint { diff --git a/pepper-sync/src/witness.rs b/pepper-sync/src/witness.rs index ffa306238f..0401dbd456 100644 --- a/pepper-sync/src/witness.rs +++ b/pepper-sync/src/witness.rs @@ -9,6 +9,7 @@ use incrementalmerkletree::{ use orchard::tree::MerkleHashOrchard; use sapling_crypto::Node; use shardtree::LocatedPrunableTree; +use zcash_client_backend::data_api::anchor_retention::{AnchorRetention, AnchorRetentionInterval}; use zcash_primitives::{block::BlockHash, merkle_tree::read_commitment_tree}; use zcash_protocol::consensus::BlockHeight; use zingo_netutils::lightwallet_protocol::TreeState; @@ -371,3 +372,129 @@ pub(crate) fn get_ironwood_tree( ) } } + +/// How many grid intervals behind the wallet's newest scanned block anchor-boundary checkpoints +/// stay pinned: the ZIP 318 anchor age cap plus two intervals of slack. +pub(crate) const ANCHOR_RETENTION_INTERVALS: u32 = zcash_protocol::zip318::ANCHOR_AGE_CAP + 2; + +/// The anchor-checkpoint retention policy in force for `consensus_parameters` — the ZIP 318 +/// grid from the Ironwood pool's activation — or `None` when that upgrade never activates. +pub(crate) fn anchor_retention_policy( + consensus_parameters: &impl zcash_protocol::consensus::Parameters, +) -> Option { + crate::wallet::PoolActivation::of(consensus_parameters, zcash_protocol::ShieldedPool::Ironwood) + .map(|activation| { + AnchorRetention::new(activation.height(), AnchorRetentionInterval::default()) + }) +} + +/// The height range whose grid-boundary checkpoints must currently be pinned, given `as_of`, +/// the wallet's newest scanned block. +pub(crate) fn anchor_retention_window( + policy: &AnchorRetention, + as_of: BlockHeight, +) -> std::ops::RangeInclusive { + let span = policy + .intervals() + .iter() + .map(|interval| interval.block_count().get()) + .max() + .unwrap_or(0) + .saturating_mul(ANCHOR_RETENTION_INTERVALS); + as_of.saturating_sub(span)..=as_of +} + +/// Re-derives one shard store's pinned-checkpoint set from `policy`, pinning every grid +/// boundary in `window` and releasing every pin outside it. +pub(crate) fn repin_anchor_checkpoints( + policy: &AnchorRetention, + window: &std::ops::RangeInclusive, + store: &mut S, +) where + S: ShardStore, +{ + for id in store.retained_checkpoints().expect("infallible") { + if !window.contains(&id) || !policy.retains(id) { + store.remove_retained_checkpoint(&id).expect("infallible"); + } + } + for boundary in policy.retained_in_range(window.clone()) { + store.add_retained_checkpoint(boundary).expect("infallible"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::shardtree_ext::ShardTreeExt; + use crate::wallet::empty_shard_tree; + + /// Pinned grid boundaries survive checkpoint pruning until re-deriving the pins releases + /// them. + #[test] + fn pinned_boundaries_survive_pruning_until_released() { + let policy = + AnchorRetention::new(BlockHeight::from_u32(1), AnchorRetentionInterval::default()); + let mut tree = empty_shard_tree::< + sapling_crypto::Node, + { sapling_crypto::NOTE_COMMITMENT_TREE_DEPTH }, + SHARD_HEIGHT, + >(); + + for height in 1..=600u32 { + let height = BlockHeight::from_u32(height); + if policy.retains(height) { + tree.store_mut() + .add_retained_checkpoint(height) + .expect("infallible"); + } + tree.append_checkpoint(height).expect("infallible"); + } + + let checkpoint_exists = |store: &shardtree::store::memory::MemoryShardStore< + sapling_crypto::Node, + BlockHeight, + >, + height: u32| { + store + .get_checkpoint(&BlockHeight::from_u32(height)) + .expect("infallible") + .is_some() + }; + let count = tree.store().checkpoint_count().expect("infallible"); + tree.store() + .for_each_checkpoint(count, |id, _| { + if u32::from(*id) < 494 { + assert_eq!( + u32::from(*id) % 144, + 0, + "old checkpoint {id} is not a boundary" + ); + } + Ok(()) + }) + .expect("infallible"); + for boundary in [144, 288, 432, 576] { + assert!( + checkpoint_exists(tree.store(), boundary), + "boundary {boundary}" + ); + } + for pruned in [143, 300, 431] { + assert!( + !checkpoint_exists(tree.store(), pruned), + "non-boundary {pruned}" + ); + } + + let window = anchor_retention_window(&policy, BlockHeight::from_u32(1200)); + assert_eq!(*window.start(), BlockHeight::from_u32(336)); + repin_anchor_checkpoints(&policy, &window, tree.store_mut()); + tree.append_checkpoint(BlockHeight::from_u32(601)) + .expect("infallible"); + assert!(!checkpoint_exists(tree.store(), 144)); + assert!(!checkpoint_exists(tree.store(), 288)); + assert!(checkpoint_exists(tree.store(), 432)); + assert!(checkpoint_exists(tree.store(), 576)); + } +} diff --git a/zingolib/src/lightclient/mock_chain_tests.rs b/zingolib/src/lightclient/mock_chain_tests.rs index c9eba0ad2a..4f97bba819 100644 --- a/zingolib/src/lightclient/mock_chain_tests.rs +++ b/zingolib/src/lightclient/mock_chain_tests.rs @@ -9,9 +9,12 @@ //! live versions are never removed. They eventually move to a gated //! "pre-migration" mod once side-by-side equivalence is documented). +use pepper_sync::sync::SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE; use pepper_sync::wallet::IronwoodNote; +use shardtree::store::ShardStore; use zcash_protocol::PoolType; use zcash_protocol::ShieldedPool; +use zcash_protocol::consensus::BlockHeight; use crate::check_client_balances; use crate::testutils::lightclient::{from_inputs, get_base_address}; @@ -1294,3 +1297,133 @@ async fn switching_the_mixnet_off_reports_the_clearnet_route() { ); } } + +#[tokio::test] +async fn shardtree_roundtrip_restores_retained_checkpoints() { + fn checkpoint_exists(store: &S, height: u32) -> bool + where + S: ShardStore, + { + store + .get_checkpoint(&BlockHeight::from_u32(height)) + .expect("infallible") + .is_some() + } + + fn all_checkpoints_stored(store: &S, chain_height: u32) -> bool + where + S: ShardStore, + { + for boundary in [144, 288] { + if !checkpoint_exists(store, boundary) { + return false; + } + } + for rolling_window in + (chain_height - SHARDTREE_CHECKPOINT_ROLLING_WINDOW_SIZE + 1)..=chain_height + { + if !checkpoint_exists(store, rolling_window) { + return false; + } + } + + true + } + + fn all_boundaries_retained(store: &S, chain_height: u32) -> bool + where + S: ShardStore, + { + let retained = store.retained_checkpoints().unwrap(); + let no_of_boundaries = chain_height / 144; + for boundary_index in 0..no_of_boundaries { + let boundary = (boundary_index + 1) * 144; + if !retained.contains(&BlockHeight::from_u32(boundary)) { + return false; + } + } + + true + } + + let mut chain_height = 500; + let mut net = MockNet::launch().await; + let mut recipient = net + .client(zingo_test_vectors::seeds::HOSPITAL_MUSEUM_SEED) + .await; + + net.chain.write().await.mine_empty_blocks(chain_height - 2); + recipient.sync_and_await().await.unwrap(); + + // create shielded note commitments to trigger checkpoint pruning + let recipient_ua = + get_base_address(&recipient, PoolType::Shielded(ShieldedPool::Orchard)).await; + fund(&net, vec![(&recipient_ua, 100_000, None)], 1).await; + recipient.sync_and_await().await.unwrap(); + + { + let shard_trees = &recipient.wallet().read().await.shard_trees; + assert!(all_checkpoints_stored( + shard_trees.sapling.store(), + chain_height + )); + assert!(all_checkpoints_stored( + shard_trees.orchard.store(), + chain_height + )); + assert!(all_checkpoints_stored( + shard_trees.ironwood.store(), + chain_height + )); + assert!(all_boundaries_retained( + shard_trees.sapling.store(), + chain_height + )); + assert!(all_boundaries_retained( + shard_trees.orchard.store(), + chain_height + )); + assert!(all_boundaries_retained( + shard_trees.ironwood.store(), + chain_height + )); + } + + recipient.save_task().await; + recipient.wait_for_save().await; + recipient.shutdown_save_task().await.unwrap(); + drop(recipient); + + let mut reloaded_recipient = net.client_from_file(0).await; + // create shielded note commitments to trigger checkpoint pruning on first sync of reloaded client + fund(&net, vec![(&recipient_ua, 100_000, None)], 1).await; + chain_height += 2; + reloaded_recipient.sync_and_await().await.unwrap(); + { + let shard_trees = &reloaded_recipient.wallet().read().await.shard_trees; + assert!(all_checkpoints_stored( + shard_trees.sapling.store(), + chain_height + )); + assert!(all_checkpoints_stored( + shard_trees.orchard.store(), + chain_height + )); + assert!(all_checkpoints_stored( + shard_trees.ironwood.store(), + chain_height + )); + assert!(all_boundaries_retained( + shard_trees.sapling.store(), + chain_height + )); + assert!(all_boundaries_retained( + shard_trees.orchard.store(), + chain_height + )); + assert!(all_boundaries_retained( + shard_trees.ironwood.store(), + chain_height + )); + } +} diff --git a/zingolib/src/testutils/mock_indexer.rs b/zingolib/src/testutils/mock_indexer.rs index ed83fcbb16..3e067eec15 100644 --- a/zingolib/src/testutils/mock_indexer.rs +++ b/zingolib/src/testutils/mock_indexer.rs @@ -837,7 +837,7 @@ pub struct MockNet { /// The fabricated chain the server reads and the test mutates. pub chain: Arc>, indexer_uri: http::Uri, - _wallet_dirs: Vec, + wallet_dirs: Vec, _server: tokio::task::JoinHandle<()>, } @@ -869,7 +869,7 @@ impl MockNet { Self { chain, indexer_uri, - _wallet_dirs: Vec::new(), + wallet_dirs: Vec::new(), _server: server, } } @@ -891,7 +891,7 @@ impl MockNet { }) .build() .unwrap(); - self._wallet_dirs.push(wallet_dir); + self.wallet_dirs.push(wallet_dir); let mut lightclient = LightClient::new(config, true) .await .expect("mock-net client construction succeeds"); @@ -918,6 +918,38 @@ impl MockNet { .await; lightclient } + + /// Builds a `LightClient` from wallet file saved in wallet directory at position `client_index` in `self.wallet_dirs`. + #[allow(unused_mut)] + pub async fn client_from_file(&mut self, client_index: usize) -> LightClient { + let wallet_dir = self + .wallet_dirs + .get(client_index) + .expect("client at given index should exist"); + let config = ClientConfig::builder() + .set_chain_type(ChainType::Regtest(ActivationHeights::default())) + .set_indexer_uri(self.indexer_uri.clone()) + .set_wallet_dir(wallet_dir.path().to_path_buf()) + .set_wallet_config(WalletConfig::Read) + .build() + .unwrap(); + let mut lightclient = LightClient::new(config, true) + .await + .expect("mock-net client construction succeeds"); + // Mock-net clients run with Mixnet Mode switched on, so every + // chain-mock send walks the fail-closed route resolver and the + // escalation orchestration instead of quietly consenting to clearnet. + // The address is never dialed: the transmit path pairs this slot + // state with arms that submit over the mock indexer's channel. + // Without the nym feature there is no mixnet and sends stay + // clearnet, so the same tests cover both routes across the + // feature matrix. + #[cfg(feature = "nym")] + lightclient + .switch_on_mixnet_for_tests(crate::mocks::transmission::MOCK_SOCKS5_ADDR) + .await; + lightclient + } } /// Builds (without transmitting) one real transaction from a synthetic diff --git a/zingolib/src/wallet/send.rs b/zingolib/src/wallet/send.rs index da0979a232..35c8966307 100644 --- a/zingolib/src/wallet/send.rs +++ b/zingolib/src/wallet/send.rs @@ -99,6 +99,20 @@ impl LightWallet { /// Whether this wallet can materialize `protocol`'s note commitment tree root, and witnesses to it, as of `height`. pub(crate) fn anchor_is_computable(&self, protocol: ShieldedPool, height: BlockHeight) -> bool { self.shards_are_scanned(protocol, None, height) + && self.checkpoint_is_retained(protocol, height) + } + + /// Whether `protocol`'s shard tree retains a checkpoint at `height`. + fn checkpoint_is_retained(&self, protocol: ShieldedPool, height: BlockHeight) -> bool { + use shardtree::store::ShardStore; + + match protocol { + ShieldedPool::Sapling => self.shard_trees.sapling.store().get_checkpoint(&height), + ShieldedPool::Orchard => self.shard_trees.orchard.store().get_checkpoint(&height), + ShieldedPool::Ironwood => self.shard_trees.ironwood.store().get_checkpoint(&height), + } + .expect("memory shard store is infallible") + .is_some() } /// Whether every shard carrying `protocol` notes between `note_height` (the scan floor when absent) and `anchor_height` is scanned.