Continuous sync

Cargo.toml

@@ -82,6 +82,7 @@ zcash_primitives = { version = "0.30.0", features = ["non-standard-fees"] }
 zcash_proofs = "0.30.0"
 zcash_protocol = "0.10.4" # public API (TODO: remove from public API)
 zcash_transparent = "0.10.0"
+zcash_script = "0.4.5"
 
 ### Blockchain Protocol
 bip0039 = { version = "0.14", features = ["rand"] }

libtonode-tests/tests/concrete.rs

@@ -1690,7 +1690,6 @@ async fn mine_to_transparent_coinbase_maturity() {
 }
 
 mod testnet_test {
-    use pepper_sync::sync_status;
     use zingo_test_vectors::seeds::HOSPITAL_MUSEUM_SEED;
     use zingolib::{
         config::{ChainType, ClientConfig, WalletConfig},
@@ -1730,11 +1729,9 @@ mod testnet_test {
             let mut interval = tokio::time::interval(std::time::Duration::from_millis(100));
             interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
             interval.tick().await;
-            while sync_status(&*lightclient.wallet().read().await)
-                .await
-                .unwrap()
-                .percentage_total_outputs_scanned
-                > 1.0
+            while lightclient
+                .latest_sync_status()
+                .is_none_or(|status| status.percentage_total_outputs_scanned < 1.0)
             {
                 interval.tick().await;
             }

libtonode-tests/tests/sync.rs

@@ -1,6 +1,7 @@
 use std::{num::NonZeroU32, time::Duration};
 
 use incrementalmerkletree::Position;
+use pepper_sync::error::SyncError;
 use pepper_sync::sync::ScanPriority;
 use pepper_sync::test_support::block;
 use pepper_sync::wallet::ShardTrees;
@@ -15,6 +16,8 @@ use zingo_test_vectors::seeds::HOSPITAL_MUSEUM_SEED;
 use zingolib::config::{ChainType, ClientConfig, WalletConfig};
 use zingolib::data::PollReport;
 use zingolib::lightclient::DEFAULT_REQUEST_TIMEOUT;
+use zingolib::lightclient::error::LightClientError;
+use zingolib::sync::SyncModeError;
 use zingolib::testutils::default_test_wallet_settings;
 use zingolib::testutils::lightclient::from_inputs::quick_send;
 use zingolib::testutils::paths::get_cargo_manifest_dir;
@@ -32,57 +35,7 @@ use zingolib_testutils::scenarios::{
     self, IndexerConvergence, increase_height_and_wait_for_client,
 };
 
-#[ignore = "temporary mainnet test for sync development"]
-#[tokio::test]
-async fn sync_mainnet_test() {
-    zingolib::ensure_default_crypto_provider();
-    tracing_subscriber::fmt().init();
-
-    let uri = construct_indexer_uri(SYNC_TEST_INDEXER.to_string()).unwrap();
-    let temp_dir = TempDir::new().unwrap();
-    let temp_path = temp_dir.path().to_path_buf();
-    let config = ClientConfig::builder()
-        .set_indexer_uri(uri.clone())
-        .set_chain_type(ChainType::Mainnet)
-        .set_wallet_dir(temp_path)
-        .set_wallet_config(WalletConfig::MnemonicPhrase {
-            mnemonic_phrase: HOSPITAL_MUSEUM_SEED.to_string(),
-            no_of_accounts: NonZeroU32::try_from(1).expect("hard-coded integer"),
-            birthday: 1_500_000,
-            wallet_settings: default_test_wallet_settings(),
-        })
-        .build()
-        .unwrap();
-    let mut lightclient = LightClient::new(config, true).await.unwrap();
-
-    lightclient.sync().await.unwrap();
-    let mut interval = tokio::time::interval(zingo_netutils::time::test::SETTLE_POLL_INTERVAL);
-    loop {
-        interval.tick().await;
-        {
-            let wallet = lightclient.wallet().read().await;
-            tracing::info!(
-                "{}",
-                json::JsonValue::from(pepper_sync::sync_status(&*wallet).await.unwrap())
-            );
-            tracing::info!("WALLET DEBUG:");
-            tracing::info!("uas: {}", wallet.unified_addresses().len());
-            tracing::info!("taddrs: {}", wallet.transparent_addresses().len());
-            tracing::info!("blocks: {}", wallet.wallet_blocks.len());
-            tracing::info!("txs: {}", wallet.wallet_transactions.len());
-            tracing::info!("nullifiers o: {}", wallet.nullifier_map.orchard.len());
-            tracing::info!("nullifiers s: {}", wallet.nullifier_map.sapling.len());
-            tracing::info!("outpoints: {}", wallet.outpoint_map.len());
-        }
-        lightclient.flush().await.unwrap();
-    }
-
-    // let wallet = lightclient.wallet.read().await;
-    // dbg!(&wallet.wallet_blocks);
-    // dbg!(&wallet.nullifier_map);
-    // dbg!(&wallet.sync_state);
-}
-
+// TODO: migrate to mock test as this test is long and connects to mainnet
 #[tokio::test]
 async fn add_subtree_roots() {
     fn assert_subtree_roots_match_server(
@@ -230,11 +183,19 @@ async fn add_subtree_roots() {
         tokio::time::sleep(Duration::from_secs(1)).await;
     }
     let _ = lightclient.stop_sync();
-    let _ = lightclient.await_sync().await;
+    match lightclient.await_sync().await {
+        Ok(_) => {}
+        Err(LightClientError::SyncError(SyncError::SyncModeError(
+            SyncModeError::SyncNotRunning,
+        )))
+        | Err(LightClientError::SyncNotRunning) => {}
+        Err(e) => {
+            panic!("{e}");
+        }
+    }
 
     {
         let shard_trees = &mut lightclient.wallet().write().await.shard_trees;
-
         assert_subtree_roots_match_server(
             shard_trees,
             sapling_subtree_roots_server.clone(),
@@ -258,6 +219,9 @@ async fn add_subtree_roots() {
         assert!(orchard_shard_addrs.len() != orchard_subtree_roots_server.len());
     }
 
+    // must wait for a new block to be mined to trigger get_subtree_roots in sync
+    tokio::time::sleep(Duration::from_secs(100)).await;
+
     lightclient.sync().await.unwrap();
     while !(lightclient
         .wallet()
@@ -272,7 +236,16 @@ async fn add_subtree_roots() {
         tokio::time::sleep(Duration::from_secs(1)).await;
     }
     let _ = lightclient.stop_sync();
-    let _ = lightclient.await_sync().await;
+    match lightclient.await_sync().await {
+        Ok(_) => {}
+        Err(LightClientError::SyncError(SyncError::SyncModeError(
+            SyncModeError::SyncNotRunning,
+        )))
+        | Err(LightClientError::SyncNotRunning) => {}
+        Err(e) => {
+            panic!("{e}");
+        }
+    }
 
     {
         let shard_trees = &mut lightclient.wallet().write().await.shard_trees;

pepper-sync/Cargo.toml

@@ -39,6 +39,7 @@ zcash_note_encryption.workspace = true
 zcash_primitives.workspace = true
 zcash_protocol.workspace = true
 zcash_transparent.workspace = true
+zcash_script.workspace = true
 
 # Protocol
 bip32.workspace = true

pepper-sync/src/client/fetch.rs

@@ -102,27 +102,27 @@ where
 {
     match fetch_request {
         FetchRequest::ChainTip(sender) => {
-            tracing::debug!("Fetching chain tip.");
+            tracing::info!("Fetching chain tip.");
             let block_id = get_latest_block(client).await;
             let _ignore_error = sender.send(block_id);
         }
         FetchRequest::CompactBlock(sender, block_height) => {
-            tracing::debug!("Fetching compact block. {:?}", &block_height);
+            tracing::info!("Fetching compact block. {:?}", &block_height);
             let block = get_block(client, block_height).await;
             let _ignore_error = sender.send(block);
         }
         FetchRequest::CompactBlockRange(sender, block_range) => {
-            tracing::debug!("Fetching compact blocks. {:?}", &block_range);
+            tracing::info!("Fetching compact blocks. {:?}", &block_range);
             let block_stream = get_block_range(client, block_range).await;
             let _ignore_error = sender.send(block_stream);
         }
         FetchRequest::NullifierRange(sender, block_range) => {
-            tracing::debug!("Fetching nullifiers. {:?}", &block_range);
+            tracing::info!("Fetching nullifiers. {:?}", &block_range);
             let block_stream = get_block_range_nullifiers(client, block_range).await;
             let _ignore_error = sender.send(block_stream);
         }
         FetchRequest::SubtreeRoots(sender, start_index, shielded_protocol, max_entries) => {
-            tracing::debug!(
+            tracing::info!(
                 "Fetching subtree roots. start index: {}. shielded protocol: {}",
                 start_index,
                 shielded_protocol
@@ -132,17 +132,17 @@ where
             let _ignore_error = sender.send(subtree_roots);
         }
         FetchRequest::TreeState(sender, block_height) => {
-            tracing::debug!("Fetching tree state. {:?}", &block_height);
+            tracing::info!("Fetching tree state. {:?}", &block_height);
             let tree_state = get_tree_state(client, block_height).await;
             let _ignore_error = sender.send(tree_state);
         }
         FetchRequest::Transaction(sender, txid) => {
-            tracing::debug!("Fetching transaction. {:?}", txid);
+            tracing::info!("Fetching transaction. {:?}", txid);
             let transaction = get_transaction(client, txid).await;
             let _ignore_error = sender.send(transaction);
         }
         FetchRequest::UtxoMetadata(sender, (addresses, start_height)) => {
-            tracing::debug!(
+            tracing::info!(
                 "Fetching unspent transparent output metadata from {:?} for addresses:\n{:?}",
                 &start_height,
                 &addresses
@@ -151,7 +151,7 @@ where
             let _ignore_error = sender.send(utxo_metadata);
         }
         FetchRequest::TransparentAddressTxs(sender, (address, block_range)) => {
-            tracing::debug!(
+            tracing::info!(
                 "Fetching raw transactions in block range {:?} for address {:?}",
                 &block_range,
                 &address

pepper-sync/src/config.rs

@@ -86,12 +86,21 @@ pub struct SyncConfig {
     pub transparent_address_discovery: TransparentAddressDiscovery,
     /// Performance level
     pub performance_level: PerformanceLevel,
+    /// Shutdown on completion
+    ///
+    /// If not set, sync will not shutdown until the consumer sets the `SyncMode` to `Shutdown` variant.
+    /// The sync engine will regularly check for new blocks mined so the wallet will always be updated to the state
+    /// of the latest chain.
+    ///
+    /// If set, sync will still check for any newly mined blocks during scanning. But when the wallet is completely
+    /// up-to-date with the latest chain, sync will shutdown.
+    pub shutdown_on_completion: bool,
 }
 
 #[cfg(feature = "wallet_essentials")]
 impl SyncConfig {
     fn serialized_version() -> u8 {
-        1
+        2
     }
 
     /// Deserialize into `reader`
@@ -101,10 +110,16 @@ impl SyncConfig {
         let gap_limit = reader.read_u8()?;
         let scopes = reader.read_u8()?;
         let performance_level = if version >= 1 {
-            PerformanceLevel::read(reader)?
+            PerformanceLevel::read(&mut reader)?
         } else {
             PerformanceLevel::High
         };
+        let shutdown_on_completion = if version >= 2 {
+            reader.read_u8()? != 0
+        } else {
+            false
+        };
+
         Ok(Self {
             transparent_address_discovery: TransparentAddressDiscovery {
                 gap_limit,
@@ -115,6 +130,7 @@ impl SyncConfig {
                 },
             },
             performance_level,
+            shutdown_on_completion,
         })
     }
 
@@ -133,7 +149,8 @@ impl SyncConfig {
             scopes |= 0b100;
         }
         writer.write_u8(scopes)?;
-        self.performance_level.write(writer)?;
+        self.performance_level.write(&mut writer)?;
+        writer.write_u8(self.shutdown_on_completion as u8)?;
 
         Ok(())
     }

pepper-sync/src/error.rs

@@ -245,7 +245,7 @@ pub enum ScanError {
     /// Continuity error.
     #[error("continuity error")]
     ContinuityError(#[from] ContinuityError),
-    /// Zcash client backend scan error
+    /// Invalid encoding.
     #[error(transparent)]
     EncodingError(#[from] EncodingInvalid),
     /// Invalid sapling nullifier
@@ -298,6 +298,17 @@ pub enum ScanError {
     /// Failed to parse encoded address.
     #[error("failed to parse encoded address")]
     AddressParseError(#[from] zcash_address::unified::ParseError),
+    /// Compact transaction contained transparent output with value outside the valid zatoshi range.
+    #[error(
+        "compact transaction contained transparent output with value {0} which is outside the valid zatoshi range"
+    )]
+    TransparentOutputInvalidValue(u64),
+    /// All transparent addresses are already in use.
+    #[error("all transparent addresses are already in use")]
+    AllAddressesInUse,
+    /// Transparent address derivation error.
+    #[error("transparent address derivation error. {0}")]
+    TransparentAddressDerivationError(bip32::Error),
 }
 
 /// The encoding of a compact Sapling output or compact Orchard action was invalid.