fix(pepper-sync): retain ZIP 318 anchor checkpoints

pepper-sync/src/sync.rs

@@ -62,6 +62,13 @@ pub mod truncate;
 /// its boundary, this constant is the one place that follows it.
 pub const MAX_REORG_ALLOWANCE: u32 = 100;
 
+/// The maximum number of checkpoint boundaries to retain.
+pub const MAX_BOUNDARY_CHECKPOINTS: u32 = 6;
+
+/// The maximum total number of checkpoints a shard tree persists: the reorg window plus the
+/// retained boundary checkpoints.
+pub const MAX_SHARDTREE_CHECKPOINTS: u32 = MAX_REORG_ALLOWANCE + MAX_BOUNDARY_CHECKPOINTS;
+
 const VERIFY_BLOCK_RANGE_SIZE: u32 = 10;
 
 /// A snapshot of the current state of sync. Useful for displaying the status of sync to a user / consumer.
@@ -446,6 +453,8 @@ where
         consensus_parameters,
     )?;
 
+    repin_anchor_checkpoints(consensus_parameters, &mut *wallet.write().await)?;
+
     let ufvks = wallet
         .read()
         .await
@@ -1726,8 +1735,9 @@ where
     H: incrementalmerkletree::Hashable + Clone + PartialEq,
 {
     let mut truncation_height = None;
+    let checkpoint_count = tree.store().checkpoint_count().expect("infallible");
     tree.store()
-        .for_each_checkpoint((MAX_REORG_ALLOWANCE + 1) as usize, |height, _| {
+        .for_each_checkpoint(checkpoint_count, |height, _| {
             if truncation_height.is_some() {
                 return Ok(());
             }
@@ -2030,6 +2040,7 @@ where
             fetch_request_sender,
             scan_range,
             highest_scanned_height,
+            witness::anchor_retention_policy(consensus_parameters),
             sapling_located_trees,
             orchard_located_trees,
             ironwood_located_trees,
@@ -2309,6 +2320,36 @@ where
     Ok(())
 }
 
+/// Re-derives the pinned anchor-checkpoint set of each shard tree from the retention policy in
+/// force for `consensus_parameters`, as of the wallet's newest scanned block.
+fn repin_anchor_checkpoints<W>(
+    consensus_parameters: &impl consensus::Parameters,
+    wallet: &mut W,
+) -> Result<(), SyncError<W::Error>>
+where
+    W: SyncWallet + SyncShardTrees,
+{
+    let Some(policy) = witness::anchor_retention_policy(consensus_parameters) else {
+        return Ok(());
+    };
+    let Some(highest_scanned_height) = wallet
+        .get_sync_state()
+        .map_err(SyncError::WalletError)?
+        .highest_scanned_height()
+    else {
+        return Ok(());
+    };
+    let window = witness::anchor_retention_window(&policy, highest_scanned_height);
+    let shard_trees = wallet
+        .get_shard_trees_mut()
+        .map_err(SyncError::WalletError)?;
+    witness::repin_anchor_checkpoints(&policy, &window, shard_trees.sapling.store_mut());
+    witness::repin_anchor_checkpoints(&policy, &window, shard_trees.orchard.store_mut());
+    witness::repin_anchor_checkpoints(&policy, &window, shard_trees.ironwood.store_mut());
+
+    Ok(())
+}
+
 async fn add_initial_frontier<W>(
     consensus_parameters: &impl consensus::Parameters,
     fetch_request_sender: mpsc::UnboundedSender<FetchRequest>,

pepper-sync/src/wallet.rs

@@ -40,7 +40,7 @@
     keys::{self, KeyId, transparent::TransparentAddressId},
     scan::compact_blocks::calculate_block_tree_bounds,
     shardtree_ext::{CheckpointAppendOutcome, ShardTreeExt as _},
-    sync::{MAX_REORG_ALLOWANCE, ScanPriority, ScanRange},
+    sync::{MAX_REORG_ALLOWANCE, MAX_SHARDTREE_CHECKPOINTS, ScanPriority, ScanRange},
     utils::{block, transaction},
     witness,
 };
@@ -1828,7 +1828,10 @@
 where
     H: incrementalmerkletree::Hashable + Clone + PartialEq,
 {
-    let mut tree = ShardTree::new(MemoryShardStore::empty(), MAX_REORG_ALLOWANCE as usize);
+    let mut tree = ShardTree::new(
+        MemoryShardStore::empty(),
+        MAX_SHARDTREE_CHECKPOINTS as usize,
+    );
 
     // `NotAboveNewest` is impossible on an empty checkpoint store.
     assert_eq!(

pepper-sync/src/wallet/serialization.rs

@@ -35,7 +35,7 @@ use crate::{
         KeyId, decode_unified_address,
         transparent::{TransparentAddressId, TransparentScope},
     },
-    sync::{MAX_REORG_ALLOWANCE, ScanPriority, ScanRange},
+    sync::{MAX_REORG_ALLOWANCE, MAX_SHARDTREE_CHECKPOINTS, ScanPriority, ScanRange},
     wallet::ScanTarget,
 };
 
@@ -1341,8 +1341,8 @@ impl ShardTrees {
                 Ok(())
             })
             .expect("Infallible");
-        if checkpoints.len() > MAX_REORG_ALLOWANCE as usize {
-            let keep_from = checkpoints.len() - MAX_REORG_ALLOWANCE as usize;
+        if checkpoints.len() > MAX_SHARDTREE_CHECKPOINTS as usize {
+            let keep_from = checkpoints.len() - MAX_SHARDTREE_CHECKPOINTS as usize;
             checkpoints.drain(..keep_from);
         }
         write_with_error_handling!(write_checkpoints, checkpoints);
@@ -1467,6 +1467,112 @@ mod tests {
         assert_eq!(recovered.ironwood_final_tree_size, 6);
     }
 
+    /// The checkpoint set of a synced wallet decomposes into exactly two parts:
+    /// [`MAX_REORG_ALLOWANCE`] rolling checkpoints, which serve ordinary reorg handling and
+    /// near-tip spends, plus [`MAX_BOUNDARY_CHECKPOINTS`] pinned ZIP 318 grid boundaries,
+    /// which serve pool crossings. Their sum is [`MAX_SHARDTREE_CHECKPOINTS`].
+    ///
+    /// The two parts are disjoint and independently bounded: pinning a boundary must not
+    /// consume a rolling slot (that would shrink the reorg window), and the rolling budget must
+    /// not displace a boundary (that would break crossings).
+    #[test]
+    fn checkpoint_set_is_reorg_window_plus_pinned_boundaries() {
+        use crate::shardtree_ext::ShardTreeExt as _;
+        use crate::sync::MAX_BOUNDARY_CHECKPOINTS;
+        use crate::witness::{anchor_retention_window, repin_anchor_checkpoints};
+        use zcash_client_backend::data_api::anchor_retention::{
+            AnchorRetention, AnchorRetentionInterval,
+        };
+
+        const TIP: u32 = 100_000;
+        const INTERVAL: u32 = 144;
+        let policy = AnchorRetention::new(
+            BlockHeight::from_u32(90_000),
+            AnchorRetentionInterval::default(),
+        );
+        let mut shard_trees = ShardTrees::new();
+
+        for height in (TIP - 2000)..=TIP {
+            let height = BlockHeight::from_u32(height);
+            let window = anchor_retention_window(&policy, height);
+            repin_anchor_checkpoints(&policy, &window, shard_trees.orchard.store_mut());
+            shard_trees
+                .orchard
+                .append_checkpoint(height)
+                .expect("infallible");
+        }
+
+        let store = shard_trees.orchard.store();
+        let total = store.checkpoint_count().expect("infallible");
+        let pinned_ids = store.retained_checkpoints().expect("infallible");
+        let mut pinned = Vec::new();
+        let mut rolling = Vec::new();
+        store
+            .for_each_checkpoint(total, |id, _| {
+                if pinned_ids.contains(id) {
+                    pinned.push(u32::from(*id));
+                } else {
+                    rolling.push(u32::from(*id));
+                }
+                Ok(())
+            })
+            .expect("infallible");
+
+        assert!(
+            pinned.iter().all(|height| height % INTERVAL == 0),
+            "every pinned checkpoint must be a grid boundary, got {pinned:?}"
+        );
+        assert_eq!(rolling.last().copied(), Some(TIP));
+        // TODO: this fails with:
+        // left:  (106, 6, 112)
+        // right: (100, 6, 106)
+        assert_eq!(
+            (rolling.len(), pinned.len(), total),
+            (
+                MAX_REORG_ALLOWANCE as usize,
+                MAX_BOUNDARY_CHECKPOINTS as usize,
+                MAX_SHARDTREE_CHECKPOINTS as usize,
+            ),
+            "(rolling, pinned, total): the pinned boundaries must be part of the \
+             MAX_SHARDTREE_CHECKPOINTS total, not stored on top of it"
+        );
+
+        for height in (TIP - MAX_REORG_ALLOWANCE + 1)..=TIP {
+            assert!(
+                store
+                    .get_checkpoint(&BlockHeight::from_u32(height))
+                    .expect("infallible")
+                    .is_some(),
+                "reorg window is missing height {height}"
+            );
+        }
+
+        assert!(
+            rolling
+                .iter()
+                .all(|height| *height >= TIP - MAX_REORG_ALLOWANCE),
+            "a rolling checkpoint survived below the reorg window: {rolling:?}"
+        );
+
+        let mut bytes = Vec::new();
+        shard_trees.write(&mut bytes).expect("write should succeed");
+        let reloaded = ShardTrees::read(bytes.as_slice()).expect("read should succeed");
+        let reloaded_store = reloaded.orchard.store();
+        for boundary in &pinned {
+            assert!(
+                reloaded_store
+                    .get_checkpoint(&BlockHeight::from_u32(*boundary))
+                    .expect("infallible")
+                    .is_some(),
+                "boundary {boundary} lost on reload; a crossing anchored there cannot be built"
+            );
+        }
+        assert_eq!(
+            reloaded_store.checkpoint_count().expect("infallible"),
+            MAX_SHARDTREE_CHECKPOINTS as usize
+        );
+    }
+
     #[test]
     fn shardtree_roundtrip_keeps_newest_checkpoints() {
         let mut shard_trees = ShardTrees::new();
@@ -1498,19 +1604,25 @@ mod tests {
         let sapling_store = roundtripped.sapling.store();
         let orchard_store = roundtripped.orchard.store();
 
-        assert_eq!(sapling_store.checkpoint_count().expect("infallible"), 100);
-        assert_eq!(orchard_store.checkpoint_count().expect("infallible"), 100);
+        assert_eq!(
+            sapling_store.checkpoint_count().expect("infallible"),
+            MAX_SHARDTREE_CHECKPOINTS as usize
+        );
+        assert_eq!(
+            orchard_store.checkpoint_count().expect("infallible"),
+            MAX_SHARDTREE_CHECKPOINTS as usize
+        );
         assert_eq!(
             sapling_store.min_checkpoint_id().expect("infallible"),
-            Some(BlockHeight::from_u32(51))
+            Some(BlockHeight::from_u32(45))
         );
         assert_eq!(
             sapling_store.max_checkpoint_id().expect("infallible"),
             Some(BlockHeight::from_u32(150))
         );
         assert_eq!(
             orchard_store.min_checkpoint_id().expect("infallible"),
-            Some(BlockHeight::from_u32(51))
+            Some(BlockHeight::from_u32(45))
         );
         assert_eq!(
             orchard_store.max_checkpoint_id().expect("infallible"),
@@ -1524,9 +1636,68 @@ mod tests {
         );
         assert!(
             sapling_store
-                .get_checkpoint(&BlockHeight::from_u32(50))
+                .get_checkpoint(&BlockHeight::from_u32(44))
+                .expect("infallible")
+                .is_none()
+        );
+    }
+
+    /// A pinned anchor checkpoint must survive the write cap. The pinned set itself is not
+    /// persisted, being re-derived at the start of every sync.
+    #[test]
+    fn shardtree_roundtrip_keeps_pinned_anchor_checkpoints() {
+        let mut shard_trees = ShardTrees::new();
+
+        for height in 1..=150 {
+            let height = BlockHeight::from_u32(height);
+            shard_trees
+                .sapling
+                .store_mut()
+                .add_checkpoint(
+                    height,
+                    Checkpoint::from_parts(TreeState::Empty, BTreeSet::new()),
+                )
+                .expect("infallible");
+        }
+        shard_trees
+            .sapling
+            .store_mut()
+            .add_retained_checkpoint(BlockHeight::from_u32(24))
+            .expect("infallible");
+
+        let mut bytes = Vec::new();
+        shard_trees.write(&mut bytes).expect("write should succeed");
+        let roundtripped = ShardTrees::read(bytes.as_slice()).expect("read should succeed");
+
+        let sapling_store = roundtripped.sapling.store();
+        assert_eq!(
+            sapling_store.checkpoint_count().expect("infallible"),
+            MAX_SHARDTREE_CHECKPOINTS as usize
+        );
+        assert!(
+            sapling_store
+                .get_checkpoint(&BlockHeight::from_u32(24))
+                .expect("infallible")
+                .is_some(),
+            "pinned checkpoint must survive the write cap"
+        );
+        assert!(
+            sapling_store
+                .get_checkpoint(&BlockHeight::from_u32(44))
                 .expect("infallible")
                 .is_none()
         );
+        assert!(
+            sapling_store
+                .get_checkpoint(&BlockHeight::from_u32(45))
+                .expect("infallible")
+                .is_some()
+        );
+        assert!(
+            sapling_store
+                .retained_checkpoints()
+                .expect("infallible")
+                .is_empty()
+        );
     }
 }