From 7559170b7193dea625feeb23fc21ea5991b53513 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Wed, 26 Aug 2026 16:48:33 -0300 Subject: [PATCH 01/43] refactor(rust-client): split both syncs into fetch and apply phases --- CHANGELOG.md | 4 + crates/rust-client/src/note/import.rs | 258 ++++++++++++++- crates/rust-client/src/note/mod.rs | 1 + .../src/note/note_update_tracker.rs | 18 ++ crates/rust-client/src/note_transport/mod.rs | 300 ++++++++++++++---- crates/rust-client/src/sync/block_header.rs | 35 ++ crates/rust-client/src/sync/mod.rs | 155 ++++++--- crates/rust-client/src/sync/state_sync.rs | 229 ++++++++++--- 8 files changed, 852 insertions(+), 148 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b03cbb29e..fdbab67d0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,8 +32,12 @@ * [BREAKING][rust] `Client::remove_address` and `Store::remove_address` return `bool` instead of `()`, reporting whether the address was tracked. When it wasn't, `Client::remove_address` now leaves the derived note tag in place instead of running its cleanup. * [BREAKING][type][rust] Added the `TransactionRequestError::ForeignProcedureInputsTooLong` variant ([#2187](https://github.com/0xMiden/rust-sdk/pull/2187)). +* [BREAKING][behavior][rust] `Client::sync_state` now fetches the Note Transport Layer pages and the node's chain data concurrently, and writes both afterwards, instead of running a full note transport sync before the chain sync. Notes delivered over the transport are still checked for consumption in the same call — the nullifier check runs after both fetches and covers them — but the chain sync's note-tag set is now read before they are written, so a tag first registered by this call's transport import is not part of this call's `SyncNotes` query. The transport path queries the node for those notes itself, so only other notes sharing that tag wait for the next sync. +* [BREAKING][behavior][rust] The note transport sync no longer persists its progress incrementally. The relay outbox, the imported notes, the backfill's covered-tag set and the transport cursor are all written after every page has been fetched, so a failure part way through leaves nothing written and the next sync re-fetches. Imports dedupe by note id, so a redo is harmless. + ### Enhancements +* [FEATURE][rust] Split both syncs into a network phase and a write phase, so every RPC call happens before the first store write and neither phase modifies the partial MMR until the writes: `Client::fetch_chain_updates` returns a `ChainSyncData` holding everything the node reported, `ChainSyncData::fetch_nullifiers` extends the nullifier check to notes another sync path delivered, and `Client::apply_chain_updates` verifies the data against the MMR and persists it. `StateSync::sync_state` is now a wrapper over the equivalent `StateSync::fetch_state` and `StateSync::build_update`. * [FEATURE][rust] Added `ChainAnchor` with `Client::execute_transaction_at` and `Client::chain_anchor_for_request` to capture and execute against a pinned reference block instead of the sync height, so a transaction summary signed at one block — which binds the reference block commitment since protocol 0.16 — can be reproduced and executed later on any client ([#2421](https://github.com/0xMiden/rust-sdk/pull/2421)). * [FEATURE][rust] A client that only watches a public account now recovers notes the account consumed authenticated, even when it never tracked them by tag. During sync it reads the note references the node attaches to the account's transactions, fetches each note body by id, and surfaces it through `InputNoteReader`. Requires node `0.15.1` ([#2300](https://github.com/0xMiden/rust-sdk/pull/2300)). * [FEATURE][cli] Added a `--payback-note-type` option to `swap` so the payback note can be created as public or private (defaults to private). Public payback works without any off-band advice now that SWAP derives the payback recipient deterministically ([#2190](https://github.com/0xMiden/rust-sdk/pull/2190)). diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 1f7e43803d..e13e12493d 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -12,7 +12,8 @@ use alloc::collections::{BTreeMap, BTreeSet}; use alloc::string::ToString; use alloc::vec::Vec; -use miden_protocol::block::BlockNumber; +use miden_protocol::block::{BlockHeader, BlockNumber}; +use miden_protocol::crypto::merkle::MerklePath; use miden_protocol::note::{ Note, NoteAttachments, @@ -28,7 +29,7 @@ use miden_tx::auth::TransactionAuthenticator; use crate::rpc::domain::note::{FetchedNote, ResolvedNoteContent, SyncedNote}; use crate::rpc::{NoteContentFetch, RpcError}; use crate::store::input_note_states::ExpectedNoteState; -use crate::store::{InputNoteRecord, InputNoteState, NoteFilter}; +use crate::store::{InputNoteRecord, InputNoteState, NoteFilter, StoreError}; use crate::sync::NoteTagRecord; use crate::{Client, ClientError}; @@ -422,7 +423,7 @@ where /// reconstructing the id from the committed metadata: `NoteId::new(details_commitment, /// metadata)`. async fn sync_expected_notes( - &mut self, + &self, request_block_num: BlockNumber, // Expected notes' details commitments with their tags. expected_notes: Vec<(NoteDetailsCommitment, NoteTag)>, @@ -470,6 +471,257 @@ where } } +/// Fetch-only expected-note import. +/// +/// These methods are the counterpart of the [`NoteFile::ExpectedNote`] path in +/// [`Client::import_notes`], split so every network call happens before the first write: +/// `fetch_expected_note_imports` and `fetch_note_block_proofs` only read and fetch, +/// `apply_expected_note_import` only writes. Used by the note transport sync, which needs its +/// network phase to overlap with the chain sync's. +impl Client +where + AUTH: TransactionAuthenticator + Sync + 'static, +{ + /// Builds the records for a batch of expected notes without writing anything. + /// + /// Each request is a note's details, the block from which its commitment should be looked for, + /// and the tag to track it under. Records for notes the node has not committed are final. + /// Records for committed notes come back pending, since their state transition also needs the + /// header of the block that committed them — [`Client::fetch_note_block_proofs`] fetches those + /// and finishes the records. + /// + /// # Errors + /// + /// - If a note being imported is currently being processed by a local transaction. + pub(crate) async fn fetch_expected_note_imports( + &self, + requests: &[(NoteDetails, BlockNumber, NoteTag)], + ) -> Result { + let mut import = ExpectedNoteImport::default(); + if requests.is_empty() { + return Ok(import); + } + + // Deduplicate by details commitment, keeping the last request for each note. + let mut requests_by_commitment = BTreeMap::new(); + for (details, after_block_num, tag) in requests { + requests_by_commitment + .insert(details.commitment(), (details.clone(), *after_block_num, *tag)); + } + + let previous_by_commitment: BTreeMap = self + .get_input_notes(NoteFilter::DetailsCommitments( + requests_by_commitment.keys().copied().collect(), + )) + .await? + .into_iter() + .map(|note| (note.details_commitment(), note)) + .collect(); + + // Validate before building anything, so a single in-flight note aborts the whole import. + for previous_note in previous_by_commitment.values() { + ensure_not_processing(Some(previous_note))?; + } + + let mut lowest_request_block: BlockNumber = u32::MAX.into(); + let mut note_requests = Vec::with_capacity(requests_by_commitment.len()); + for (commitment, (_, after_block_num, tag)) in &requests_by_commitment { + note_requests.push((*commitment, *tag)); + lowest_request_block = lowest_request_block.min(*after_block_num); + } + let mut committed_notes_data = + self.sync_expected_notes(lowest_request_block, note_requests).await?; + + for (commitment, (details, after_block_num, tag)) in requests_by_commitment { + let mut note_record = + previous_by_commitment.get(&commitment).cloned().unwrap_or_else(|| { + InputNoteRecord::new( + details, + NoteAttachments::empty(), + self.store.get_current_timestamp(), + ExpectedNoteState { + metadata: None, + after_block_num, + tag: Some(tag), + } + .into(), + ) + }); + + // Notes the node has not reported as committed keep their expected record untouched. + let Some(SyncedNote { committed: committed_note, content }) = + committed_notes_data.remove(&commitment) + else { + import.notes.push(note_record); + continue; + }; + + let attachments = content + .map(ResolvedNoteContent::into_attachments) + .filter(|attachments| !attachments.is_empty()); + + let metadata = *committed_note.metadata(); + let mut changed = note_record + .inclusion_proof_received(committed_note.inclusion_proof().clone(), metadata)?; + + if let Some(attachments) = attachments { + changed |= note_record.attachments_received(attachments); + } + + import.pending.push(PendingBlockHeaderNote { + record: note_record, + block_num: committed_note.block_num(), + committed_tag: metadata.tag(), + changed, + }); + } + + Ok(import) + } + + /// Fetches the header and MMR proof of every block that committed one of the notes in + /// `imports`, then finishes the records waiting on them. + /// + /// Each block is fetched once even when it committed several notes, in one page or across + /// pages. A block the client's partial MMR already tracks has its header read from the store + /// and needs no insert, so it is absent from the returned map. + /// + /// Nothing is written and the MMR is not modified: tracking the returned headers and inserting + /// them is [`Client::apply_blocks`]'s job, so the proof paths are verified against the peaks + /// only once, next to the writes they authenticate. + pub(crate) async fn fetch_note_block_proofs( + &self, + imports: &mut [ExpectedNoteImport], + ) -> Result, ClientError> { + let requested_blocks: BTreeSet = imports + .iter() + .flat_map(|import| import.pending.iter().map(|note| note.block_num)) + .collect(); + + if requested_blocks.is_empty() { + return Ok(BTreeMap::new()); + } + + let partial_mmr = self.get_current_partial_mmr().await?; + + let mut headers = BTreeMap::new(); + let mut blocks_to_insert = BTreeMap::new(); + for block_num in requested_blocks { + if partial_mmr.is_tracked(block_num.as_usize()) { + let (block_header, _) = self + .store + .get_block_header_by_num(block_num) + .await? + .ok_or(StoreError::BlockHeaderNotFound(block_num))?; + headers.insert(block_num, block_header); + continue; + } + + let (block_header, mmr_proof) = + self.rpc_api.get_block_header_with_proof(block_num).await?; + headers.insert(block_num, block_header.clone()); + blocks_to_insert.insert(block_num, (block_header, mmr_proof.merkle_path().clone())); + } + + for import in imports { + for mut pending in core::mem::take(&mut import.pending) { + let block_header = headers + .get(&pending.block_num) + .expect("every pending note's block was fetched above"); + + // `block_header_received` transitions the record's state, so it must always run. + let changed = + pending.changed | pending.record.block_header_received(block_header)?; + + if changed { + // Once committed, the note no longer needs its expected-note tag. + import.tags_to_remove.push(NoteTagRecord::with_note_source( + pending.committed_tag, + pending.record.details_commitment(), + )); + import.notes.push(pending.record); + } + } + } + + Ok(blocks_to_insert) + } + + /// Writes an [`ExpectedNoteImport`], returning the details commitments of the written records. + /// + /// Block headers are not written here: [`Client::apply_blocks`] must run first so a record is + /// never persisted as committed before the header proving its inclusion. + /// + /// # Panics + /// + /// Panics if the import still has records waiting on a block header. + pub(crate) async fn apply_expected_note_import( + &mut self, + import: ExpectedNoteImport, + ) -> Result, ClientError> { + assert!( + import.pending.is_empty(), + "block proofs must be fetched before an expected-note import is applied" + ); + + for tag in import.tags_to_remove { + self.store.remove_note_tag(tag).await?; + } + + let mut written = Vec::with_capacity(import.notes.len()); + for note in import.notes { + let details_commitment = note.details_commitment(); + if let InputNoteState::Expected(ExpectedNoteState { tag: Some(tag), .. }) = note.state() + { + self.store + .add_note_tag(NoteTagRecord::with_note_source(*tag, details_commitment)) + .await?; + } + self.store.upsert_input_notes(&[note]).await?; + written.push(details_commitment); + } + + Ok(written) + } +} + +// EXPECTED NOTE IMPORT +// ================================================================================================ + +/// A record whose inclusion proof and attachments have been applied, waiting for the header of the +/// block that committed it. +struct PendingBlockHeaderNote { + /// The record, with every transition but the block header already applied. + record: InputNoteRecord, + /// Block that committed the note. + block_num: BlockNumber, + /// Note-source tag to drop once the record is committed. + committed_tag: NoteTag, + /// Whether the inclusion-proof and attachment transitions already changed the record. + changed: bool, +} + +/// Everything an expected-note import needs to write, with nothing written yet. +/// +/// Built by [`Client::fetch_expected_note_imports`], completed by +/// [`Client::fetch_note_block_proofs`] and written by [`Client::apply_expected_note_import`]. +#[derive(Default)] +pub(crate) struct ExpectedNoteImport { + /// Records ready to write. + notes: Vec, + /// Records waiting on a block header. + pending: Vec, + /// Note-source tags to remove. + tags_to_remove: Vec, +} + +impl ExpectedNoteImport { + /// The records this import is about to write. + pub(crate) fn input_note_records(&self) -> impl Iterator { + self.notes.iter() + } +} + // HELPERS // ================================================================================================ diff --git a/crates/rust-client/src/note/mod.rs b/crates/rust-client/src/note/mod.rs index dac79af457..195a2d580c 100644 --- a/crates/rust-client/src/note/mod.rs +++ b/crates/rust-client/src/note/mod.rs @@ -69,6 +69,7 @@ use crate::store::{InputNoteRecord, NoteFilter, OutputNoteRecord}; use crate::{Client, ClientError, IdPrefixFetchError}; mod import; +pub(crate) use import::ExpectedNoteImport; mod note_reader; mod note_screener; mod note_update_tracker; diff --git a/crates/rust-client/src/note/note_update_tracker.rs b/crates/rust-client/src/note/note_update_tracker.rs index d47f8e9dc3..ab999d7829 100644 --- a/crates/rust-client/src/note/note_update_tracker.rs +++ b/crates/rust-client/src/note/note_update_tracker.rs @@ -377,6 +377,24 @@ impl NoteUpdateTracker { }) } + /// Tracks additional already-persisted input notes as unmodified context. + /// + /// Used to extend a sync's nullifier check to notes that are about to be written by another + /// path (e.g. the note transport sync) and are therefore absent from the store snapshot this + /// tracker was built from. Notes already tracked for the same details commitment are skipped, + /// so a record built by this sync is never replaced by a stale one. + pub(crate) fn track_existing_input_notes( + &mut self, + notes: impl IntoIterator, + ) { + for note in notes { + if self.input_notes.contains_key(¬e.details_commitment()) { + continue; + } + self.insert_input_note(note, NoteUpdateType::None); + } + } + /// Appends nullifiers to the per-account ordered nullifier list. /// /// Nullifiers from the same account must be in execution order; ordering across different diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index dd3c84e72b..b774a132a3 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -8,13 +8,15 @@ use alloc::collections::{BTreeMap, BTreeSet}; use alloc::string::String; use alloc::sync::Arc; use alloc::vec::Vec; +use core::slice; use futures::Stream; use miden_protocol::address::Address; -use miden_protocol::block::BlockNumber; +use miden_protocol::block::{BlockHeader, BlockNumber}; +use miden_protocol::crypto::merkle::MerklePath; +use miden_protocol::crypto::merkle::mmr::PartialMmr; use miden_protocol::note::{Note, NoteDetails, NoteDetailsCommitment, NoteHeader, NoteId, NoteTag}; use miden_protocol::utils::serde::Serializable; -use miden_standards::note::{NoteFile, NoteSyncHint}; use miden_tx::auth::TransactionAuthenticator; use miden_tx::utils::serde::{ ByteReader, @@ -25,6 +27,8 @@ use miden_tx::utils::serde::{ }; pub use self::errors::NoteTransportError; +use crate::note::ExpectedNoteImport; +use crate::store::InputNoteRecord; use crate::sync::NoteTagSource; use crate::{Client, ClientError}; @@ -173,11 +177,33 @@ impl Client { /// error, so a relay failure can't block a sync. Callers driving retries /// themselves can invoke it directly and inspect the returned error. pub async fn flush_relay_outbox(&self) -> Result<(), ClientError> { + let (remaining, last_err) = self.retry_relay_outbox().await?; + + if let Some(remaining) = remaining { + self.save_relay_outbox(remaining).await?; + } + + if let Some(err) = last_err { + return Err(err.into()); + } + Ok(()) + } + + /// Re-sends every relay payload in the durable outbox, returning the entries that still failed + /// and the last error, without writing anything. + /// + /// The returned entries are `None` when the outbox was empty and there is therefore nothing to + /// persist; `Some` (possibly empty) means the outbox must be overwritten with them. + /// [`Client::flush_relay_outbox`] does that write; the note transport sync defers it to its + /// apply phase. + async fn retry_relay_outbox( + &self, + ) -> Result<(Option>, Option), ClientError> { let api = self.get_note_transport_api()?; let entries = self.load_relay_outbox().await?; if entries.is_empty() { - return Ok(()); + return Ok((None, None)); } // Attempt every entry independently so a single persistently-failing @@ -208,12 +234,7 @@ impl Client { } } - self.save_relay_outbox(remaining).await?; - - if let Some(err) = last_err { - return Err(err.into()); - } - Ok(()) + Ok((Some(remaining), last_err)) } /// Load the durable relay outbox. @@ -348,17 +369,29 @@ where /// fetches only notes past the stored cursor. Historical notes for a newly tracked tag are /// recovered automatically by [`Client::sync_note_transport`], which backfills each new tag. pub async fn fetch_private_notes(&mut self) -> Result<(), ClientError> { + self.ensure_genesis_in_place().await?; + let note_tags: Vec = self.store.get_unique_note_tags().await?.into_iter().collect(); let cursor = self.store.get_note_transport_cursor().await?; - let (_, new_cursor) = self.fetch_transport_notes(cursor, ¬e_tags).await?; + let mut id_by_commitment = BTreeMap::new(); + let (mut import, new_cursor) = + self.fetch_transport_page(cursor, ¬e_tags, &mut id_by_commitment).await?; + let blocks = self.fetch_note_block_proofs(slice::from_mut(&mut import)).await?; + + let mut partial_mmr = self.get_current_partial_mmr().await?; + self.apply_blocks(blocks, &mut partial_mmr).await?; + self.apply_expected_note_import(import).await?; + self.cache_partial_mmr(partial_mmr).await?; + self.store.update_note_transport_cursor(new_cursor).await?; Ok(()) } - /// Backfill historical private notes for tags added after the global cursor advanced. + /// Plans the backfill of historical private notes for tags added after the global cursor + /// advanced. /// /// The global transport cursor is shared across all tracked tags and only moves forward, so a /// tag that starts being tracked late never sees its notes that already sit below the cursor. @@ -370,48 +403,50 @@ where /// the steady-state stream is harmless. /// /// At most [`Self::MAX_BACKFILL_TAGS_PER_SYNC`] tags are backfilled per call; any remainder - /// stays uncovered and is picked up on the next sync. Returns the ids of notes imported here. - pub(crate) async fn backfill_new_tags(&mut self) -> Result, ClientError> { + /// stays uncovered and is picked up on the next sync. + /// + /// Returns the pruned covered set, whether pruning changed it, and the tags to backfill. + /// Reads only: persisting the covered set is left to the apply phase, which writes it after + /// the imported notes so a crash re-backfills instead of skipping a tag whose notes were + /// never written. + async fn plan_backfill(&self) -> Result<(BTreeSet, bool, Vec), ClientError> { let candidates = self.backfill_candidate_tags().await?; let loaded = self.load_covered_tags().await?; // Drop tags no longer tracked. Keeping a removed tag marked covered would make a later // re-add skip its backlog, silently missing notes that arrived while it was untracked. - let mut covered: BTreeSet = loaded.intersection(&candidates).copied().collect(); - if covered.len() != loaded.len() { - self.save_covered_tags(&covered).await?; - } - - let new_tags: Vec = candidates.difference(&covered).copied().collect(); + let covered: BTreeSet = loaded.intersection(&candidates).copied().collect(); + let pruned = covered.len() != loaded.len(); - let mut imported_ids = Vec::new(); - for tag in new_tags.into_iter().take(Self::MAX_BACKFILL_TAGS_PER_SYNC) { - imported_ids.extend(self.backfill_tag(tag).await?); - covered.insert(tag); - // Persist after each tag so a crash mid-backfill keeps completed tags covered. A redo - // is harmless because imports dedupe; the dangerous direction (marking covered before - // the import lands) never happens. - self.save_covered_tags(&covered).await?; - } + let new_tags: Vec = candidates + .difference(&covered) + .copied() + .take(Self::MAX_BACKFILL_TAGS_PER_SYNC) + .collect(); - Ok(imported_ids) + Ok((covered, pruned, new_tags)) } /// Drain a single tag's full history from the transport, paging until the cursor stops /// advancing. Uses a local cursor and never touches the global one, so it cannot regress - /// steady-state progress. Returns the ids of the notes it imported. - async fn backfill_tag(&mut self, tag: NoteTag) -> Result, ClientError> { - let mut imported_ids = Vec::new(); + /// steady-state progress. Returns one import per fetched page, none of them written. + async fn backfill_tag( + &self, + tag: NoteTag, + id_by_commitment: &mut BTreeMap, + ) -> Result, ClientError> { + let mut imports = Vec::new(); let mut cursor = NoteTransportCursor::init(); for _ in 0..Self::MAX_BACKFILL_ITERATIONS { - let (ids, new_cursor) = self.fetch_transport_notes(cursor, &[tag]).await?; - imported_ids.extend(ids); + let (import, new_cursor) = + self.fetch_transport_page(cursor, &[tag], id_by_commitment).await?; + imports.push(import); // Terminate on any lack of forward progress. A well-behaved server returns // `new_cursor == cursor` when there are no new notes for this tag (since // `rcursor = max(cursor, max_seq_returned)`); using `<=` also handles implementations // that return an `init()` cursor on empty batches (see the in-tree mock transport). if new_cursor <= cursor { - return Ok(imported_ids); + return Ok(imports); } cursor = new_cursor; } @@ -421,23 +456,26 @@ where ))) } - /// Fetch one batch of notes from the note transport network for the provided tags. + /// Fetch one batch of notes from the note transport network for the provided tags and build + /// the records they imply, without writing anything. /// - /// The server paginates; this method issues one RPC and returns the imported details - /// commitments together with the new cursor. The returned cursor equals the input cursor when - /// the batch was empty (i.e. no new notes). Callers that want to drain a tag's full backlog - /// should loop until `new_cursor == cursor` (see [`Client::backfill_new_tags`]). Callers that - /// do steady-state polling (see [`Client::sync_state`] / [`Client::fetch_private_notes`]) - /// should call this once per tick with the stored cursor. + /// The server paginates; this method issues one RPC and returns the import together with the + /// new cursor. The returned cursor equals the input cursor when the batch was empty (i.e. no + /// new notes). Callers that want to drain a tag's full backlog should loop until + /// `new_cursor == cursor` (see [`Client::backfill_tag`]). Callers that do steady-state polling + /// (see [`Client::sync_state`] / [`Client::fetch_private_notes`]) should call this once per + /// tick with the stored cursor. /// - /// Downloaded notes are imported into the local store. Persistence of the returned cursor is - /// left to the caller so that drain loops can guard against regression of an already-advanced - /// stored cursor. - pub(crate) async fn fetch_transport_notes( - &mut self, + /// Each downloaded note's id is recorded in `id_by_commitment` so the caller can resolve the + /// written records back to note ids once the final record set is known. Persistence of the + /// returned cursor is left to the caller so that drain loops can guard against regression of + /// an already-advanced stored cursor. + async fn fetch_transport_page( + &self, cursor: NoteTransportCursor, tags: &[NoteTag], - ) -> Result<(Vec, NoteTransportCursor), ClientError> { + id_by_commitment: &mut BTreeMap, + ) -> Result<(ExpectedNoteImport, NoteTransportCursor), ClientError> { // Fallback lookback window, in blocks, used only for notes the transport delivered // without a sender-provided block hint. Scanning back from sync height handles // the race where a note is committed on-chain just before the NTL delivers its data. @@ -449,7 +487,6 @@ where // TODO: perhaps we should not need to map received IDs with details commitments, and // instead we may allow `InputNoteRecord` to optionally keep NoteIds. Then within // `import_note` we could match everything by ID and remove this map check - let mut id_by_commitment: BTreeMap = BTreeMap::new(); let (note_infos, rcursor) = self.get_note_transport_api()?.fetch_notes(tags, cursor).await?; for note_info in ¬e_infos { @@ -468,24 +505,165 @@ where let fallback_after_block_num = BlockNumber::from(sync_height.as_u32().saturating_sub(NOTE_LOOKBACK_BLOCKS)); - let mut note_requests = Vec::with_capacity(notes.len()); + let mut requests = Vec::with_capacity(notes.len()); for (note, block_hint) in notes { let tag = note.metadata().tag(); // Prefer the sender-provided hint, falling back to the lookback window when absent. let after_block_num = block_hint.unwrap_or(fallback_after_block_num); - let note_file = NoteFile::ExpectedNote { - details: note.into(), - sync_hint: NoteSyncHint::new(after_block_num, tag), - }; - note_requests.push(note_file); + requests.push((NoteDetails::from(note), after_block_num, tag)); } - let imported_commitments = self.import_notes(¬e_requests).await?; - let imported_ids = imported_commitments - .into_iter() - .filter_map(|commitment| id_by_commitment.get(&commitment).copied()) - .collect(); - Ok((imported_ids, rcursor)) + let import = self.fetch_expected_note_imports(&requests).await?; + + Ok((import, rcursor)) + } + + /// Fetches everything the note transport sync is about to write, without writing any of it. + /// + /// Runs the relay-outbox retries, the per-tag history backfill and the steady-state page, and + /// returns them as a [`NoteTransportSyncData`] for [`Client::apply_note_transport_updates`]. + /// Takes `&self` so it can run concurrently with the chain sync's fetch phase. + /// + /// The block proofs of the notes reported as committed are *not* fetched here: they are a + /// second network pass over this result (see [`Client::fetch_note_block_proofs`]), because the + /// blocks to prove are only known once every page has been fetched. + /// + /// Returns empty data when note transport is not configured. + pub(crate) async fn fetch_note_transport_updates( + &self, + ) -> Result { + let mut data = NoteTransportSyncData::default(); + if !self.is_note_transport_enabled() { + return Ok(data); + } + + // Re-send any private notes whose previous relay attempt failed. A relay error is logged, + // not propagated: a failing relay must not block the sync, and the entries stay durable + // for the next attempt. + match self.retry_relay_outbox().await { + Ok((outbox_remaining, last_err)) => { + if let Some(err) = last_err { + tracing::warn!(?err, "relay outbox flush failed during sync; entries retained"); + } + data.outbox_remaining = outbox_remaining; + }, + Err(err) => { + tracing::warn!(?err, "relay outbox flush failed during sync; entries retained"); + }, + } + + // Recover historical private notes for any tag added after the global cursor advanced. + // This drains each newly tracked tag from the start, fetching only that tag's own history. + let (mut covered, pruned, new_tags) = self.plan_backfill().await?; + let backfilled = !new_tags.is_empty(); + for tag in new_tags { + data.imports.extend(self.backfill_tag(tag, &mut data.id_by_commitment).await?); + covered.insert(tag); + } + if pruned || backfilled { + data.covered_tags = Some(covered); + } + + let cursor = self.store.get_note_transport_cursor().await?; + let note_tags: Vec = + self.store.get_unique_note_tags().await?.into_iter().collect(); + let (import, new_cursor) = self + .fetch_transport_page(cursor, ¬e_tags, &mut data.id_by_commitment) + .await?; + data.imports.push(import); + data.cursor = Some(new_cursor); + + Ok(data) + } + + /// Writes everything [`Client::fetch_note_transport_updates`] fetched, returning the ids of + /// the imported notes. + /// + /// The notes are written before the covered-tag set and the cursor, so a crash between them + /// re-fetches instead of skipping notes that were never written. `partial_mmr` is loaded and + /// cached by the caller, so one MMR can be shared with the chain sync's apply phase. + /// + /// [`Client::fetch_note_block_proofs`] must have run on `data` first, otherwise the records + /// waiting on a block header are still pending and the import panics. + pub(crate) async fn apply_note_transport_updates( + &mut self, + data: NoteTransportSyncData, + partial_mmr: &mut PartialMmr, + ) -> Result, ClientError> { + let NoteTransportSyncData { + outbox_remaining, + covered_tags, + imports, + blocks, + id_by_commitment, + cursor, + } = data; + + if let Some(outbox_remaining) = outbox_remaining { + self.save_relay_outbox(outbox_remaining).await?; + } + + self.apply_blocks(blocks, partial_mmr).await?; + + let mut imported_ids = Vec::new(); + for import in imports { + let written = self.apply_expected_note_import(import).await?; + imported_ids.extend( + written + .into_iter() + .filter_map(|commitment| id_by_commitment.get(&commitment).copied()), + ); + } + + if let Some(covered_tags) = covered_tags { + self.save_covered_tags(&covered_tags).await?; + } + + if let Some(cursor) = cursor { + self.store.update_note_transport_cursor(cursor).await?; + } + + imported_ids.sort_unstable(); + imported_ids.dedup(); + + Ok(imported_ids) + } +} + +// NOTE TRANSPORT SYNC DATA +// ================================================================================================ + +/// Everything the note transport sync is about to write, with nothing written yet. +/// +/// Built by [`Client::fetch_note_transport_updates`], completed by +/// [`Client::fetch_note_block_proofs`] and written by +/// [`Client::apply_note_transport_updates`]. +#[derive(Default)] +pub(crate) struct NoteTransportSyncData { + /// Relay entries whose re-send failed and that the outbox is overwritten with. `None` when + /// the outbox was empty and needs no write. + outbox_remaining: Option>, + /// Covered-tag set to persist, `None` when it did not change. + covered_tags: Option>, + /// One entry per fetched page, in fetch order. + pub(crate) imports: Vec, + /// Headers of the blocks that committed the fetched notes, with the MMR proof paths the node + /// returned. Filled by [`Client::fetch_note_block_proofs`]. + pub(crate) blocks: BTreeMap, + /// Note ids by details commitment, taken from the note headers the transport returned. Used + /// to resolve the written records back to ids. + id_by_commitment: BTreeMap, + /// New global cursor, from the steady-state page. `None` when no page was fetched. + cursor: Option, +} + +impl NoteTransportSyncData { + /// The records this sync is about to write. + /// + /// Used to extend the chain sync's nullifier check to the notes the transport just delivered, + /// which are not in the store yet. + pub(crate) fn input_note_records(&self) -> impl Iterator { + self.imports.iter().flat_map(ExpectedNoteImport::input_note_records) } } diff --git a/crates/rust-client/src/sync/block_header.rs b/crates/rust-client/src/sync/block_header.rs index 1abddd1c61..1f5d598a5e 100644 --- a/crates/rust-client/src/sync/block_header.rs +++ b/crates/rust-client/src/sync/block_header.rs @@ -1,3 +1,4 @@ +use alloc::collections::BTreeMap; use alloc::sync::Arc; use alloc::vec::Vec; @@ -117,6 +118,40 @@ impl Client { Ok(Rpo256::hash_elements(&elements)) } + /// Tracks each fetched block in `partial_mmr` and inserts its header together with the + /// authentication nodes that tracking produced. + /// + /// Tracking is what verifies the node's proof path against the current peaks, so every block + /// is tracked before the first insert: a path that doesn't verify fails with nothing written. + /// Blocks already tracked are skipped, which covers both a block from an earlier sync and the + /// same block reaching this call from two different pages. + /// + /// Loading the MMR and caching it afterwards is the caller's, so one MMR can be threaded + /// through several apply steps and cached once. + pub(crate) async fn apply_blocks( + &mut self, + blocks: BTreeMap, + partial_mmr: &mut PartialMmr, + ) -> Result<(), ClientError> { + let mut authenticated_blocks = Vec::with_capacity(blocks.len()); + for (block_num, (block_header, mmr_path)) in blocks { + if partial_mmr.is_tracked(block_num.as_usize()) { + continue; + } + + let path_nodes = + track_block_in_mmr(partial_mmr, block_num, block_header.commitment(), &mmr_path)?; + authenticated_blocks.push((block_header, path_nodes)); + } + + for (block_header, path_nodes) in authenticated_blocks { + let nodes = authenticated_block_nodes(&block_header, path_nodes); + self.store.insert_block_header(&block_header, &nodes, true).await?; + } + + Ok(()) + } + // HELPERS // -------------------------------------------------------------------------------------------- diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index dfed94e40a..3fdd116913 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -57,6 +57,7 @@ //! processed and applied to the local store. use alloc::collections::BTreeSet; +use alloc::format; use alloc::sync::Arc; use alloc::vec::Vec; use core::cmp::max; @@ -71,7 +72,7 @@ use miden_tx::utils::serde::{Deserializable, DeserializationError, Serializable} use tracing::{debug, info}; use crate::pswap::PswapChainObserver; -use crate::store::{NoteFilter, TransactionFilter}; +use crate::store::{InputNoteRecord, NoteFilter, TransactionFilter}; use crate::{Client, ClientError}; mod block_header; @@ -82,7 +83,8 @@ mod note_observer; pub use note_observer::NoteObserver; mod state_sync; -pub use state_sync::{NoteUpdateAction, OnNoteReceived, StateSync, StateSyncInput}; +pub(crate) use state_sync::block_num_from_forest; +pub use state_sync::{ChainSyncData, NoteUpdateAction, OnNoteReceived, StateSync, StateSyncInput}; mod state_sync_update; pub use state_sync_update::{ @@ -129,24 +131,77 @@ where /// [`Client::sync_state`] for the combined sync, or call [`Client::sync_note_transport`] /// separately. /// - /// Builds the default sync input, runs [`StateSync::sync_state`] (see that method for the - /// detailed pipeline), applies the resulting update to the store, caches the partial MMR, and - /// prunes irrelevant blocks according to the configured cadence. + /// Fetches everything from the node first ([`Client::fetch_chain_updates`] and + /// [`ChainSyncData::fetch_nullifiers`]), then applies the result + /// ([`Client::apply_chain_updates`]), caches the partial MMR, and prunes irrelevant blocks + /// according to the configured cadence. pub async fn sync_chain(&mut self) -> Result { self.ensure_genesis_in_place().await?; self.ensure_rpc_limits_in_place().await?; + let mut data = self.fetch_chain_updates().await?; + // No other sync path ran, so there are no externally delivered notes to cover. + data.fetch_nullifiers(Vec::new()).await?; + + let mut partial_mmr = self.get_current_partial_mmr().await?; + let sync_summary = self.apply_chain_updates(data, &mut partial_mmr).await?; + + // Cache MMR so pruning can reuse in-memory MMR. + self.cache_partial_mmr(partial_mmr).await?; + + self.maybe_untrack_and_prune_irrelevant_blocks().await?; + + Ok(sync_summary) + } + + /// Fetches the node's view of everything that changed since the client's chain tip, without + /// writing anything or modifying the partial MMR. + /// + /// Builds the default sync input and runs [`StateSync::fetch_state`]. The nullifier check is + /// not part of this: run [`ChainSyncData::fetch_nullifiers`] on the result before applying it, + /// so it can also cover notes another sync path delivered in the same call. + /// + /// Takes `&self` so it can run concurrently with the note transport sync's fetch phase. + pub async fn fetch_chain_updates(&self) -> Result { // Each `NoteObserver` owns its own per-sync state; `with_note_observer` just attaches. let note_screener = self.note_screener(); let state_sync = StateSync::new(self.rpc_api.clone(), Arc::new(note_screener), self.tx_discard_delta) .with_note_observer(Arc::new(PswapChainObserver::new(self.store.clone()))); + let input = self.build_sync_input().await?; + let block_from = block_num_from_forest(&self.get_current_partial_mmr().await?)?; - let mut partial_mmr = self.get_current_partial_mmr().await?; + state_sync.fetch_state(block_from, input).await + } + + /// Verifies fetched chain data against `partial_mmr` and writes the resulting update. + /// + /// `partial_mmr` is loaded and cached by the caller, so one MMR can be shared with the note + /// transport sync's apply phase; it is left advanced to the chain tip. + /// + /// # Errors + /// + /// Returns an error if `partial_mmr` no longer starts where the data was fetched from, which + /// means another sync advanced the store in between and the data is stale. + pub async fn apply_chain_updates( + &mut self, + data: ChainSyncData, + partial_mmr: &mut PartialMmr, + ) -> Result { + let block_from = block_num_from_forest(partial_mmr)?; + if block_from != data.block_from { + return Err(ClientError::ChainValidationError(format!( + "chain sync data starts at block {} but the client is at block {block_from}", + data.block_from + ))); + } - // Get the sync update from the network - let state_sync_update = state_sync.sync_state(&mut partial_mmr, input).await?; + // The observers accumulated their state during the fetch, so the hooks below must run + // against the very same instances. + let state_sync = data.state_sync.clone(); + + let state_sync_update = StateSync::build_update(data, partial_mmr)?; let sync_summary: SyncSummary = (&state_sync_update).into(); debug!(sync_summary = ?sync_summary, "Sync summary computed"); @@ -163,11 +218,6 @@ where .await .map_err(ClientError::StoreError)?; - // Cache MMR so pruning can reuse in-memory MMR. - self.cache_partial_mmr(partial_mmr).await?; - - self.maybe_untrack_and_prune_irrelevant_blocks().await?; - Ok(sync_summary) } @@ -179,42 +229,67 @@ where if !self.is_note_transport_enabled() { return Ok(Vec::new()); } + self.ensure_genesis_in_place().await?; - // Drain any private notes whose previous relay attempt failed. A flush - // error is logged, not propagated: a failing relay must not block the - // sync, and the entries stay durable for the next attempt. - if let Err(err) = self.flush_relay_outbox().await { - tracing::warn!(?err, "relay outbox flush failed during sync; entries retained"); - } - - // Recover historical private notes for any tag added after the global cursor advanced. - // This drains each newly tracked tag from the start, fetching only that tag's own history. - let mut imported_ids = self.backfill_new_tags().await?; - - let cursor = self.store.get_note_transport_cursor().await?; - let note_tags: Vec<_> = self.store.get_unique_note_tags().await?.into_iter().collect(); - let (ids, new_cursor) = self.fetch_transport_notes(cursor, ¬e_tags).await?; - self.store.update_note_transport_cursor(new_cursor).await?; - imported_ids.extend(ids); + let mut data = self.fetch_note_transport_updates().await?; + data.blocks = self.fetch_note_block_proofs(&mut data.imports).await?; - imported_ids.sort_unstable(); - imported_ids.dedup(); + let mut partial_mmr = self.get_current_partial_mmr().await?; + let imported_ids = self.apply_note_transport_updates(data, &mut partial_mmr).await?; + self.cache_partial_mmr(partial_mmr).await?; Ok(imported_ids) } - /// Runs the full client sync. + /// Runs the full client sync: private notes from the Note Transport Layer and the client's + /// on-chain state with the Miden node. /// - /// First fetches private notes from the Note Transport Layer (see - /// [`Client::sync_note_transport`]), then syncs the client's on-chain state with the Miden - /// node (see [`Client::sync_chain`]). If note transport is disabled, this is equivalent to - /// [`Client::sync_chain`]. + /// The two are fetched concurrently, since the transport pages and the node's sync data are + /// independent. Everything that touches the MMR or the store runs sequentially afterwards, so + /// the network round trips of one sync overlap with the other's while the writes stay ordered: /// - /// Fails fast on the first error. Private notes delivered via NTL are imported before the - /// chain sync reads its input set, so their nullifiers are checked in the same call. + /// 1. Concurrently: the note transport fetch phase and [`Client::fetch_chain_updates`]. + /// 2. The block proofs of the blocks that committed the delivered notes, which are only known + /// once every transport page has been fetched. + /// 3. [`ChainSyncData::fetch_nullifiers`], covering the tracked notes *and* the ones the + /// transport just delivered, so a note delivered and consumed in the same window is reported + /// as consumed by this call. + /// 4. The writes: the transport update first, since a nullified delivered note is written by + /// the chain update as an update to the row the transport insert creates. + /// + /// Fails fast on the first error: the concurrent fetch drops the other side, and neither has + /// written anything at that point. A relay re-send that already went out stays in the outbox + /// and is retried on the next sync, which the receiver dedupes by note id. + /// + /// Unlike the previous sequential behavior, the chain sync's input set is read before the + /// delivered notes are written, so a note tag registered by this call's transport import is + /// not part of this call's `sync_notes` query. The transport path queries the node for exactly + /// those notes itself, so only other notes sharing that tag wait for the next sync. pub async fn sync_state(&mut self) -> Result { - let new_private_notes = self.sync_note_transport().await?; - let mut summary = self.sync_chain().await?; + // Both fetch phases need genesis in place, and connecting here means the two concurrent + // futures never race on the RPC client's lazy connect. + self.ensure_genesis_in_place().await?; + self.ensure_rpc_limits_in_place().await?; + + let (mut transport_data, mut chain_data) = + futures::try_join!(self.fetch_note_transport_updates(), self.fetch_chain_updates())?; + + transport_data.blocks = self.fetch_note_block_proofs(&mut transport_data.imports).await?; + + let delivered_notes: Vec = + transport_data.input_note_records().cloned().collect(); + chain_data.fetch_nullifiers(delivered_notes).await?; + + let mut partial_mmr = self.get_current_partial_mmr().await?; + let new_private_notes = + self.apply_note_transport_updates(transport_data, &mut partial_mmr).await?; + let mut summary = self.apply_chain_updates(chain_data, &mut partial_mmr).await?; + + // Cache MMR so pruning can reuse in-memory MMR. + self.cache_partial_mmr(partial_mmr).await?; + + self.maybe_untrack_and_prune_irrelevant_blocks().await?; + summary.new_private_notes = new_private_notes; Ok(summary) } diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index a7a6b06f78..b050aaf8a4 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -270,23 +270,51 @@ impl StateSync { /// /// Use [`Client::build_sync_input()`](`crate::Client::build_sync_input()`) to build the default /// input, or assemble it manually for custom sync. The `current_partial_mmr` is taken by - /// mutable reference so callers can keep it in memory across syncs. + /// mutable reference so callers can keep it in memory across syncs; it is only modified once + /// every check has passed. /// - /// During the sync process, the following steps are performed: - /// 1. Fetch sync data from the node (MMR delta, note inclusions, transactions). - /// 2. Update account states (fetch updated public accounts, flag mismatched private ones). - /// 3. Advance the partial MMR to the chain tip. - /// 4. Screen note inclusions via the configured [`OnNoteReceived`] callback. - /// 5. Process transaction inclusions (commit local txs, record external consumers, discard - /// stale/expired txs, commit output notes). - /// 6. Detect consumed notes via nullifier sync (optional, see - /// [`Self::disable_nullifier_sync`]). - /// 7. Track in the MMR the screened blocks that still hold an unspent note. + /// Runs the three phases in order, each of which can also be driven separately: + /// 1. [`Self::fetch_state`] — every node call but the nullifier check. + /// 2. [`ChainSyncData::fetch_nullifiers`] — the nullifier check. + /// 3. [`Self::build_update`] — verify against the MMR and assemble the update. pub async fn sync_state( &self, current_partial_mmr: &mut PartialMmr, input: StateSyncInput, ) -> Result { + let block_num = block_num_from_forest(current_partial_mmr)?; + + let mut data = self.fetch_state(block_num, input).await?; + data.fetch_nullifiers(Vec::new()).await?; + + // Work on a clone so any validation failure leaves `current_partial_mmr` untouched. + let mut working_mmr = current_partial_mmr.clone(); + let update = Self::build_update(data, &mut working_mmr)?; + *current_partial_mmr = working_mmr; + + Ok(update) + } + + /// Fetches the node's view of everything that changed since `block_from`, without verifying it + /// against the client's MMR or writing anything. + /// + /// Runs every node call of a chain sync except the nullifier check, which + /// [`ChainSyncData::fetch_nullifiers`] performs afterwards so it can also cover notes another + /// sync path delivered in the same call. Screening the received notes reads the store and may + /// execute transactions, but nothing is persisted. + /// + /// The steps are: + /// 1. Fetch sync data from the node (MMR delta, note inclusions, transactions). + /// 2. Update account states (fetch updated public accounts, flag mismatched private ones). + /// 3. Screen note inclusions via the configured [`OnNoteReceived`] callback. + /// 4. Process transaction inclusions (commit local txs, record external consumers, discard + /// stale/expired txs, commit output notes). + /// 5. Recover the public notes the tracked accounts consumed. + pub async fn fetch_state( + &self, + block_from: BlockNumber, + input: StateSyncInput, + ) -> Result { let StateSyncInput { accounts, note_tags, @@ -294,28 +322,25 @@ impl StateSync { output_notes, uncommitted_transactions, } = input; - let block_num = u32::try_from(current_partial_mmr.forest().num_leaves().saturating_sub(1)) - .map_err(|_| ClientError::InvalidPartialMmrForest)? - .into(); let note_tags = Arc::new(note_tags); let account_ids: Vec = accounts.iter().map(AccountHeader::id).collect(); let mut note_updates = NoteUpdateTracker::new(input_notes, output_notes); let mut transaction_updates = TransactionUpdateTracker::new(uncommitted_transactions); - let mut partial_blockchain_updates = PartialBlockchainUpdates::default(); let mut account_updates = AccountUpdates::default(); - let Some(sync_data) = self.fetch_sync_data(block_num, &account_ids, ¬e_tags).await? + let Some(sync_data) = self.fetch_sync_data(block_from, &account_ids, ¬e_tags).await? else { // No progress — already at the tip. - return Ok(StateSyncUpdate::from_parts( - block_num, - partial_blockchain_updates, + return Ok(ChainSyncData { + state_sync: self.clone(), + block_from, + advance: None, note_updates, transaction_updates, account_updates, - )); + }); }; let FetchedSyncData { @@ -324,7 +349,6 @@ impl StateSync { note_blocks, transactions, } = sync_data; - let chain_tip = chain_tip_header.block_num(); let new_commitments = derive_account_commitments(&transactions); let superseded_states = self @@ -332,7 +356,7 @@ impl StateSync { &mut account_updates, &accounts, &new_commitments, - block_num, + block_from, &chain_tip_header, ) .await?; @@ -342,15 +366,6 @@ impl StateSync { transaction_updates.apply_superseded_account_state(superseded_state); } - // Work on a clone so any validation failure leaves `current_partial_mmr` untouched. - let mut working_mmr = current_partial_mmr.clone(); - - Self::advance_mmr( - mmr_delta, - &chain_tip_header, - &mut working_mmr, - &mut partial_blockchain_updates, - )?; let relevant_note_blocks = self.screen_note_blocks(note_blocks, &mut note_updates).await?; self.apply_transactions_and_nullifiers( &chain_tip_header, @@ -359,30 +374,78 @@ impl StateSync { &mut transaction_updates, )?; - if self.sync_nullifiers { - self.nullifiers_state_sync( - &mut note_updates, - &mut transaction_updates, - chain_tip, - block_num, - ) - .await?; - } - self.recover_consumed_public_notes(&mut note_updates, &transactions).await?; + Ok(ChainSyncData { + state_sync: self.clone(), + block_from, + advance: Some(ChainAdvance { + chain_tip_header, + mmr_delta, + relevant_note_blocks, + }), + note_updates, + transaction_updates, + account_updates, + }) + } + + /// Verifies the fetched chain data against `partial_mmr` and turns it into the update to + /// persist. + /// + /// This is the only step that mutates the MMR: it applies the node's delta, checks the + /// resulting peaks against the chain tip header's chain commitment, and tracks the screened + /// note blocks that still hold an unspent note. It performs no I/O, so every check runs before + /// the caller's first write, and a failure leaves `partial_mmr` to be discarded by the caller. + pub fn build_update( + data: ChainSyncData, + partial_mmr: &mut PartialMmr, + ) -> Result { + let ChainSyncData { + block_from, + advance, + note_updates, + transaction_updates, + account_updates, + .. + } = data; + + let mut partial_blockchain_updates = PartialBlockchainUpdates::default(); + + let Some(ChainAdvance { + chain_tip_header, + mmr_delta, + relevant_note_blocks, + }) = advance + else { + // No progress — already at the tip. + return Ok(StateSyncUpdate::from_parts( + block_from, + partial_blockchain_updates, + note_updates, + transaction_updates, + account_updates, + )); + }; + let chain_tip = chain_tip_header.block_num(); + + Self::advance_mmr( + mmr_delta, + &chain_tip_header, + partial_mmr, + &mut partial_blockchain_updates, + )?; + let blocks_with_unspent_notes: BTreeSet = note_updates.unspent_input_note_block_numbers().collect(); Self::validate_and_track_note_blocks( relevant_note_blocks, &blocks_with_unspent_notes, - &mut working_mmr, + partial_mmr, &mut partial_blockchain_updates, )?; - *current_partial_mmr = working_mmr; - Ok(StateSyncUpdate::from_parts( chain_tip, partial_blockchain_updates, @@ -1268,9 +1331,87 @@ impl StateSync { } } +// CHAIN SYNC DATA +// ================================================================================================ + +/// The chain data a sync fetched from the node, before any of it has been verified against the +/// client's MMR or written. +/// +/// Built by [`StateSync::fetch_state`], extended by [`Self::fetch_nullifiers`] and turned into a +/// [`StateSyncUpdate`] by [`StateSync::build_update`]. It carries the [`StateSync`] that produced +/// it, since the note observers accumulate per-note state during the fetch and drain it in their +/// apply hook, so the same instances have to survive into the apply phase. +pub struct ChainSyncData { + /// The component that produced this data. + pub(crate) state_sync: StateSync, + /// The chain tip the sync started from. + pub(crate) block_from: BlockNumber, + /// What the node reported beyond `block_from`, or `None` when the client was already at the + /// chain tip. + advance: Option, + note_updates: NoteUpdateTracker, + transaction_updates: TransactionUpdateTracker, + account_updates: AccountUpdates, +} + +/// The part of a [`ChainSyncData`] that only exists when the node reported progress. +struct ChainAdvance { + /// Header of the chain tip the sync advanced to. + chain_tip_header: BlockHeader, + /// MMR delta from `block_from` to the chain tip, excluding the chain-tip leaf. + mmr_delta: MmrDelta, + /// Screened blocks holding a client-relevant note, each with its `sync_notes` MMR path. + relevant_note_blocks: Vec, +} + +impl ChainSyncData { + /// Checks the node for nullifiers of every note this sync could have consumed. + /// + /// `extra_notes` are notes another sync path fetched in the same call and is about to write — + /// the private notes delivered over the note transport layer. They are not in the store yet, + /// so they are tracked here as existing notes: that puts their nullifiers in the query and + /// lets a hit be applied to the record that will be written, which is what makes a note + /// delivered and consumed within one sync report as consumed by that same sync. + /// + /// No-op when the nullifier sync is disabled (see [`StateSync::disable_nullifier_sync`]) or + /// when the node reported no progress, since there is no block range to query. + pub async fn fetch_nullifiers( + &mut self, + extra_notes: Vec, + ) -> Result<(), ClientError> { + if !self.state_sync.sync_nullifiers { + return Ok(()); + } + + let Some(chain_tip) = + self.advance.as_ref().map(|advance| advance.chain_tip_header.block_num()) + else { + return Ok(()); + }; + + self.note_updates.track_existing_input_notes(extra_notes); + + self.state_sync + .nullifiers_state_sync( + &mut self.note_updates, + &mut self.transaction_updates, + chain_tip, + self.block_from, + ) + .await + } +} + // HELPERS // ================================================================================================ +/// Returns the block number the given partial MMR is synced to. +pub(crate) fn block_num_from_forest(partial_mmr: &PartialMmr) -> Result { + Ok(u32::try_from(partial_mmr.forest().num_leaves().saturating_sub(1)) + .map_err(|_| ClientError::InvalidPartialMmrForest)? + .into()) +} + /// Groups transaction records by `(account_id, block_num)`. fn group_txs_by_account_block( transaction_records: &[RpcTransactionRecord], From db355b254ad55f1017f0aebd4f58173e1b4f94db Mon Sep 17 00:00:00 2001 From: ricomateo Date: Wed, 26 Aug 2026 17:39:59 -0300 Subject: [PATCH 02/43] refactor(rust-client): keep flush_relay_outbox as a single function --- CHANGELOG.md | 2 +- crates/rust-client/src/note_transport/mod.rs | 73 +++++++------------- crates/rust-client/src/sync/mod.rs | 5 +- 3 files changed, 29 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdbab67d0d..65b7270cc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ * [BREAKING][type][rust] Added the `TransactionRequestError::ForeignProcedureInputsTooLong` variant ([#2187](https://github.com/0xMiden/rust-sdk/pull/2187)). * [BREAKING][behavior][rust] `Client::sync_state` now fetches the Note Transport Layer pages and the node's chain data concurrently, and writes both afterwards, instead of running a full note transport sync before the chain sync. Notes delivered over the transport are still checked for consumption in the same call — the nullifier check runs after both fetches and covers them — but the chain sync's note-tag set is now read before they are written, so a tag first registered by this call's transport import is not part of this call's `SyncNotes` query. The transport path queries the node for those notes itself, so only other notes sharing that tag wait for the next sync. -* [BREAKING][behavior][rust] The note transport sync no longer persists its progress incrementally. The relay outbox, the imported notes, the backfill's covered-tag set and the transport cursor are all written after every page has been fetched, so a failure part way through leaves nothing written and the next sync re-fetches. Imports dedupe by note id, so a redo is harmless. +* [BREAKING][behavior][rust] The note transport sync no longer persists its progress incrementally. The imported notes, the backfill's covered-tag set and the transport cursor are all written after every page has been fetched, so a failure part way through leaves none of them written and the next sync re-fetches. Imports dedupe by note id, so a redo is harmless. The relay outbox is unchanged: `Client::flush_relay_outbox` still persists its own remaining entries as it runs. ### Enhancements diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index b774a132a3..532e0e8123 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -177,33 +177,11 @@ impl Client { /// error, so a relay failure can't block a sync. Callers driving retries /// themselves can invoke it directly and inspect the returned error. pub async fn flush_relay_outbox(&self) -> Result<(), ClientError> { - let (remaining, last_err) = self.retry_relay_outbox().await?; - - if let Some(remaining) = remaining { - self.save_relay_outbox(remaining).await?; - } - - if let Some(err) = last_err { - return Err(err.into()); - } - Ok(()) - } - - /// Re-sends every relay payload in the durable outbox, returning the entries that still failed - /// and the last error, without writing anything. - /// - /// The returned entries are `None` when the outbox was empty and there is therefore nothing to - /// persist; `Some` (possibly empty) means the outbox must be overwritten with them. - /// [`Client::flush_relay_outbox`] does that write; the note transport sync defers it to its - /// apply phase. - async fn retry_relay_outbox( - &self, - ) -> Result<(Option>, Option), ClientError> { let api = self.get_note_transport_api()?; let entries = self.load_relay_outbox().await?; if entries.is_empty() { - return Ok((None, None)); + return Ok(()); } // Attempt every entry independently so a single persistently-failing @@ -234,7 +212,12 @@ impl Client { } } - Ok((Some(remaining), last_err)) + self.save_relay_outbox(remaining).await?; + + if let Some(err) = last_err { + return Err(err.into()); + } + Ok(()) } /// Load the durable relay outbox. @@ -518,11 +501,17 @@ where Ok((import, rcursor)) } - /// Fetches everything the note transport sync is about to write, without writing any of it. + /// Fetches what the note transport sync is about to write, writing only the relay outbox. + /// + /// Runs the relay-outbox flush, the per-tag history backfill and the steady-state page, and + /// returns the latter two as a [`NoteTransportSyncData`] for + /// [`Client::apply_note_transport_updates`]. Takes `&self` so it can run concurrently with the + /// chain sync's fetch phase. /// - /// Runs the relay-outbox retries, the per-tag history backfill and the steady-state page, and - /// returns them as a [`NoteTransportSyncData`] for [`Client::apply_note_transport_updates`]. - /// Takes `&self` so it can run concurrently with the chain sync's fetch phase. + /// The outbox flush is the exception to this being a read-only phase: it persists its own + /// remaining entries rather than handing them to the apply phase. That write touches only the + /// outbox setting and is safe to redo, so it does not affect what a failure part way through + /// leaves behind for the notes, tags and cursor. /// /// The block proofs of the notes reported as committed are *not* fetched here: they are a /// second network pass over this result (see [`Client::fetch_note_block_proofs`]), because the @@ -537,19 +526,12 @@ where return Ok(data); } - // Re-send any private notes whose previous relay attempt failed. A relay error is logged, + // Drain any private notes whose previous relay attempt failed. A flush error is logged, // not propagated: a failing relay must not block the sync, and the entries stay durable - // for the next attempt. - match self.retry_relay_outbox().await { - Ok((outbox_remaining, last_err)) => { - if let Some(err) = last_err { - tracing::warn!(?err, "relay outbox flush failed during sync; entries retained"); - } - data.outbox_remaining = outbox_remaining; - }, - Err(err) => { - tracing::warn!(?err, "relay outbox flush failed during sync; entries retained"); - }, + // for the next attempt. This is the one write the fetch phase performs; it touches only + // the outbox setting, which is independent of everything the apply phase writes. + if let Err(err) = self.flush_relay_outbox().await { + tracing::warn!(?err, "relay outbox flush failed during sync; entries retained"); } // Recover historical private notes for any tag added after the global cursor advanced. @@ -583,6 +565,9 @@ where /// re-fetches instead of skipping notes that were never written. `partial_mmr` is loaded and /// cached by the caller, so one MMR can be shared with the chain sync's apply phase. /// + /// The relay outbox is not written here: [`Client::flush_relay_outbox`] persists it itself, + /// during the fetch phase. + /// /// [`Client::fetch_note_block_proofs`] must have run on `data` first, otherwise the records /// waiting on a block header are still pending and the import panics. pub(crate) async fn apply_note_transport_updates( @@ -591,7 +576,6 @@ where partial_mmr: &mut PartialMmr, ) -> Result, ClientError> { let NoteTransportSyncData { - outbox_remaining, covered_tags, imports, blocks, @@ -599,10 +583,6 @@ where cursor, } = data; - if let Some(outbox_remaining) = outbox_remaining { - self.save_relay_outbox(outbox_remaining).await?; - } - self.apply_blocks(blocks, partial_mmr).await?; let mut imported_ids = Vec::new(); @@ -640,9 +620,6 @@ where /// [`Client::apply_note_transport_updates`]. #[derive(Default)] pub(crate) struct NoteTransportSyncData { - /// Relay entries whose re-send failed and that the outbox is overwritten with. `None` when - /// the outbox was empty and needs no write. - outbox_remaining: Option>, /// Covered-tag set to persist, `None` when it did not change. covered_tags: Option>, /// One entry per fetched page, in fetch order. diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 3fdd116913..ab374dd147 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -258,8 +258,9 @@ where /// the chain update as an update to the row the transport insert creates. /// /// Fails fast on the first error: the concurrent fetch drops the other side, and neither has - /// written anything at that point. A relay re-send that already went out stays in the outbox - /// and is retried on the next sync, which the receiver dedupes by note id. + /// written anything the sync depends on. The one exception is the relay outbox, which + /// [`Client::flush_relay_outbox`] persists during the fetch; a re-send that already went out + /// stays in the outbox and is retried on the next sync, which the receiver dedupes by note id. /// /// Unlike the previous sequential behavior, the chain sync's input set is read before the /// delivered notes are written, so a note tag registered by this call's transport import is From b12aff4282b64595c3d582f7abd8384e484d6d6d Mon Sep 17 00:00:00 2001 From: ricomateo Date: Wed, 26 Aug 2026 17:59:17 -0300 Subject: [PATCH 03/43] refactor(rust-client): group the expected-note import fns in one impl block --- crates/rust-client/src/note/import.rs | 427 +++++++++++++------------- 1 file changed, 211 insertions(+), 216 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index e13e12493d..000a36bdf8 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -173,6 +173,211 @@ where Ok(imported_commitments) } + // FETCH-ONLY EXPECTED NOTE IMPORT + // -------------------------------------------------------------------------------------------- + + /// Builds the records for a batch of expected notes without writing anything. + /// + /// Each request is a note's details, the block from which its commitment should be looked for, + /// and the tag to track it under. Records for notes the node has not committed are final. + /// Records for committed notes come back pending, since their state transition also needs the + /// header of the block that committed them — [`Client::fetch_note_block_proofs`] fetches those + /// and finishes the records. + /// + /// # Errors + /// + /// - If a note being imported is currently being processed by a local transaction. + pub(crate) async fn fetch_expected_note_imports( + &self, + requests: &[(NoteDetails, BlockNumber, NoteTag)], + ) -> Result { + let mut import = ExpectedNoteImport::default(); + if requests.is_empty() { + return Ok(import); + } + + // Deduplicate by details commitment, keeping the last request for each note. + let mut requests_by_commitment = BTreeMap::new(); + for (details, after_block_num, tag) in requests { + requests_by_commitment + .insert(details.commitment(), (details.clone(), *after_block_num, *tag)); + } + + let previous_by_commitment: BTreeMap = self + .get_input_notes(NoteFilter::DetailsCommitments( + requests_by_commitment.keys().copied().collect(), + )) + .await? + .into_iter() + .map(|note| (note.details_commitment(), note)) + .collect(); + + // Validate before building anything, so a single in-flight note aborts the whole import. + for previous_note in previous_by_commitment.values() { + ensure_not_processing(Some(previous_note))?; + } + + let mut lowest_request_block: BlockNumber = u32::MAX.into(); + let mut note_requests = Vec::with_capacity(requests_by_commitment.len()); + for (commitment, (_, after_block_num, tag)) in &requests_by_commitment { + note_requests.push((*commitment, *tag)); + lowest_request_block = lowest_request_block.min(*after_block_num); + } + let mut committed_notes_data = + self.sync_expected_notes(lowest_request_block, note_requests).await?; + + for (commitment, (details, after_block_num, tag)) in requests_by_commitment { + let mut note_record = + previous_by_commitment.get(&commitment).cloned().unwrap_or_else(|| { + InputNoteRecord::new( + details, + NoteAttachments::empty(), + self.store.get_current_timestamp(), + ExpectedNoteState { + metadata: None, + after_block_num, + tag: Some(tag), + } + .into(), + ) + }); + + // Notes the node has not reported as committed keep their expected record untouched. + let Some(SyncedNote { committed: committed_note, content }) = + committed_notes_data.remove(&commitment) + else { + import.notes.push(note_record); + continue; + }; + + let attachments = content + .map(ResolvedNoteContent::into_attachments) + .filter(|attachments| !attachments.is_empty()); + + let metadata = *committed_note.metadata(); + let mut changed = note_record + .inclusion_proof_received(committed_note.inclusion_proof().clone(), metadata)?; + + if let Some(attachments) = attachments { + changed |= note_record.attachments_received(attachments); + } + + import.pending.push(PendingBlockHeaderNote { + record: note_record, + block_num: committed_note.block_num(), + committed_tag: metadata.tag(), + changed, + }); + } + + Ok(import) + } + + /// Fetches the header and MMR proof of every block that committed one of the notes in + /// `imports`, then finishes the records waiting on them. + /// + /// Each block is fetched once even when it committed several notes, in one page or across + /// pages. A block the client's partial MMR already tracks has its header read from the store + /// and needs no insert, so it is absent from the returned map. + /// + /// Nothing is written and the MMR is not modified: tracking the returned headers and inserting + /// them is [`Client::apply_blocks`]'s job, so the proof paths are verified against the peaks + /// only once, next to the writes they authenticate. + pub(crate) async fn fetch_note_block_proofs( + &self, + imports: &mut [ExpectedNoteImport], + ) -> Result, ClientError> { + let requested_blocks: BTreeSet = imports + .iter() + .flat_map(|import| import.pending.iter().map(|note| note.block_num)) + .collect(); + + if requested_blocks.is_empty() { + return Ok(BTreeMap::new()); + } + + let partial_mmr = self.get_current_partial_mmr().await?; + + let mut headers = BTreeMap::new(); + let mut blocks_to_insert = BTreeMap::new(); + for block_num in requested_blocks { + if partial_mmr.is_tracked(block_num.as_usize()) { + let (block_header, _) = self + .store + .get_block_header_by_num(block_num) + .await? + .ok_or(StoreError::BlockHeaderNotFound(block_num))?; + headers.insert(block_num, block_header); + continue; + } + + let (block_header, mmr_proof) = + self.rpc_api.get_block_header_with_proof(block_num).await?; + headers.insert(block_num, block_header.clone()); + blocks_to_insert.insert(block_num, (block_header, mmr_proof.merkle_path().clone())); + } + + for import in imports { + for mut pending in core::mem::take(&mut import.pending) { + let block_header = headers + .get(&pending.block_num) + .expect("every pending note's block was fetched above"); + + // `block_header_received` transitions the record's state, so it must always run. + let changed = + pending.changed | pending.record.block_header_received(block_header)?; + + if changed { + // Once committed, the note no longer needs its expected-note tag. + import.tags_to_remove.push(NoteTagRecord::with_note_source( + pending.committed_tag, + pending.record.details_commitment(), + )); + import.notes.push(pending.record); + } + } + } + + Ok(blocks_to_insert) + } + + /// Writes an [`ExpectedNoteImport`], returning the details commitments of the written records. + /// + /// Block headers are not written here: [`Client::apply_blocks`] must run first so a record is + /// never persisted as committed before the header proving its inclusion. + /// + /// # Panics + /// + /// Panics if the import still has records waiting on a block header. + pub(crate) async fn apply_expected_note_import( + &mut self, + import: ExpectedNoteImport, + ) -> Result, ClientError> { + assert!( + import.pending.is_empty(), + "block proofs must be fetched before an expected-note import is applied" + ); + + for tag in import.tags_to_remove { + self.store.remove_note_tag(tag).await?; + } + + let mut written = Vec::with_capacity(import.notes.len()); + for note in import.notes { + let details_commitment = note.details_commitment(); + if let InputNoteState::Expected(ExpectedNoteState { tag: Some(tag), .. }) = note.state() + { + self.store + .add_note_tag(NoteTagRecord::with_note_source(*tag, details_commitment)) + .await?; + } + self.store.upsert_input_notes(&[note]).await?; + written.push(details_commitment); + } + + Ok(written) + } + // HELPERS // ================================================================================================ @@ -471,220 +676,6 @@ where } } -/// Fetch-only expected-note import. -/// -/// These methods are the counterpart of the [`NoteFile::ExpectedNote`] path in -/// [`Client::import_notes`], split so every network call happens before the first write: -/// `fetch_expected_note_imports` and `fetch_note_block_proofs` only read and fetch, -/// `apply_expected_note_import` only writes. Used by the note transport sync, which needs its -/// network phase to overlap with the chain sync's. -impl Client -where - AUTH: TransactionAuthenticator + Sync + 'static, -{ - /// Builds the records for a batch of expected notes without writing anything. - /// - /// Each request is a note's details, the block from which its commitment should be looked for, - /// and the tag to track it under. Records for notes the node has not committed are final. - /// Records for committed notes come back pending, since their state transition also needs the - /// header of the block that committed them — [`Client::fetch_note_block_proofs`] fetches those - /// and finishes the records. - /// - /// # Errors - /// - /// - If a note being imported is currently being processed by a local transaction. - pub(crate) async fn fetch_expected_note_imports( - &self, - requests: &[(NoteDetails, BlockNumber, NoteTag)], - ) -> Result { - let mut import = ExpectedNoteImport::default(); - if requests.is_empty() { - return Ok(import); - } - - // Deduplicate by details commitment, keeping the last request for each note. - let mut requests_by_commitment = BTreeMap::new(); - for (details, after_block_num, tag) in requests { - requests_by_commitment - .insert(details.commitment(), (details.clone(), *after_block_num, *tag)); - } - - let previous_by_commitment: BTreeMap = self - .get_input_notes(NoteFilter::DetailsCommitments( - requests_by_commitment.keys().copied().collect(), - )) - .await? - .into_iter() - .map(|note| (note.details_commitment(), note)) - .collect(); - - // Validate before building anything, so a single in-flight note aborts the whole import. - for previous_note in previous_by_commitment.values() { - ensure_not_processing(Some(previous_note))?; - } - - let mut lowest_request_block: BlockNumber = u32::MAX.into(); - let mut note_requests = Vec::with_capacity(requests_by_commitment.len()); - for (commitment, (_, after_block_num, tag)) in &requests_by_commitment { - note_requests.push((*commitment, *tag)); - lowest_request_block = lowest_request_block.min(*after_block_num); - } - let mut committed_notes_data = - self.sync_expected_notes(lowest_request_block, note_requests).await?; - - for (commitment, (details, after_block_num, tag)) in requests_by_commitment { - let mut note_record = - previous_by_commitment.get(&commitment).cloned().unwrap_or_else(|| { - InputNoteRecord::new( - details, - NoteAttachments::empty(), - self.store.get_current_timestamp(), - ExpectedNoteState { - metadata: None, - after_block_num, - tag: Some(tag), - } - .into(), - ) - }); - - // Notes the node has not reported as committed keep their expected record untouched. - let Some(SyncedNote { committed: committed_note, content }) = - committed_notes_data.remove(&commitment) - else { - import.notes.push(note_record); - continue; - }; - - let attachments = content - .map(ResolvedNoteContent::into_attachments) - .filter(|attachments| !attachments.is_empty()); - - let metadata = *committed_note.metadata(); - let mut changed = note_record - .inclusion_proof_received(committed_note.inclusion_proof().clone(), metadata)?; - - if let Some(attachments) = attachments { - changed |= note_record.attachments_received(attachments); - } - - import.pending.push(PendingBlockHeaderNote { - record: note_record, - block_num: committed_note.block_num(), - committed_tag: metadata.tag(), - changed, - }); - } - - Ok(import) - } - - /// Fetches the header and MMR proof of every block that committed one of the notes in - /// `imports`, then finishes the records waiting on them. - /// - /// Each block is fetched once even when it committed several notes, in one page or across - /// pages. A block the client's partial MMR already tracks has its header read from the store - /// and needs no insert, so it is absent from the returned map. - /// - /// Nothing is written and the MMR is not modified: tracking the returned headers and inserting - /// them is [`Client::apply_blocks`]'s job, so the proof paths are verified against the peaks - /// only once, next to the writes they authenticate. - pub(crate) async fn fetch_note_block_proofs( - &self, - imports: &mut [ExpectedNoteImport], - ) -> Result, ClientError> { - let requested_blocks: BTreeSet = imports - .iter() - .flat_map(|import| import.pending.iter().map(|note| note.block_num)) - .collect(); - - if requested_blocks.is_empty() { - return Ok(BTreeMap::new()); - } - - let partial_mmr = self.get_current_partial_mmr().await?; - - let mut headers = BTreeMap::new(); - let mut blocks_to_insert = BTreeMap::new(); - for block_num in requested_blocks { - if partial_mmr.is_tracked(block_num.as_usize()) { - let (block_header, _) = self - .store - .get_block_header_by_num(block_num) - .await? - .ok_or(StoreError::BlockHeaderNotFound(block_num))?; - headers.insert(block_num, block_header); - continue; - } - - let (block_header, mmr_proof) = - self.rpc_api.get_block_header_with_proof(block_num).await?; - headers.insert(block_num, block_header.clone()); - blocks_to_insert.insert(block_num, (block_header, mmr_proof.merkle_path().clone())); - } - - for import in imports { - for mut pending in core::mem::take(&mut import.pending) { - let block_header = headers - .get(&pending.block_num) - .expect("every pending note's block was fetched above"); - - // `block_header_received` transitions the record's state, so it must always run. - let changed = - pending.changed | pending.record.block_header_received(block_header)?; - - if changed { - // Once committed, the note no longer needs its expected-note tag. - import.tags_to_remove.push(NoteTagRecord::with_note_source( - pending.committed_tag, - pending.record.details_commitment(), - )); - import.notes.push(pending.record); - } - } - } - - Ok(blocks_to_insert) - } - - /// Writes an [`ExpectedNoteImport`], returning the details commitments of the written records. - /// - /// Block headers are not written here: [`Client::apply_blocks`] must run first so a record is - /// never persisted as committed before the header proving its inclusion. - /// - /// # Panics - /// - /// Panics if the import still has records waiting on a block header. - pub(crate) async fn apply_expected_note_import( - &mut self, - import: ExpectedNoteImport, - ) -> Result, ClientError> { - assert!( - import.pending.is_empty(), - "block proofs must be fetched before an expected-note import is applied" - ); - - for tag in import.tags_to_remove { - self.store.remove_note_tag(tag).await?; - } - - let mut written = Vec::with_capacity(import.notes.len()); - for note in import.notes { - let details_commitment = note.details_commitment(); - if let InputNoteState::Expected(ExpectedNoteState { tag: Some(tag), .. }) = note.state() - { - self.store - .add_note_tag(NoteTagRecord::with_note_source(*tag, details_commitment)) - .await?; - } - self.store.upsert_input_notes(&[note]).await?; - written.push(details_commitment); - } - - Ok(written) - } -} - // EXPECTED NOTE IMPORT // ================================================================================================ @@ -703,8 +694,12 @@ struct PendingBlockHeaderNote { /// Everything an expected-note import needs to write, with nothing written yet. /// -/// Built by [`Client::fetch_expected_note_imports`], completed by -/// [`Client::fetch_note_block_proofs`] and written by [`Client::apply_expected_note_import`]. +/// The fetch-only counterpart of the [`NoteFile::ExpectedNote`] path in +/// [`Client::import_notes`], split so every network call happens before the first write: built by +/// [`Client::fetch_expected_note_imports`] and completed by +/// [`Client::fetch_note_block_proofs`], which only read and fetch, then written by +/// [`Client::apply_expected_note_import`]. Used by the note transport sync, which needs its +/// network phase to overlap with the chain sync's. #[derive(Default)] pub(crate) struct ExpectedNoteImport { /// Records ready to write. From 1fca6082d0d7bdd90dfc40510f5cc5e2a5bc76d2 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Wed, 26 Aug 2026 19:16:25 -0300 Subject: [PATCH 04/43] refactor(rust-client): make ChainSyncData plain data and move its method to StateSync --- crates/rust-client/src/sync/mod.rs | 43 +++++----- crates/rust-client/src/sync/state_sync.rs | 95 +++++++++++------------ 2 files changed, 71 insertions(+), 67 deletions(-) diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index ab374dd147..9dbd8fdcdc 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -132,19 +132,19 @@ where /// separately. /// /// Fetches everything from the node first ([`Client::fetch_chain_updates`] and - /// [`ChainSyncData::fetch_nullifiers`]), then applies the result + /// [`StateSync::fetch_nullifiers`]), then applies the result /// ([`Client::apply_chain_updates`]), caches the partial MMR, and prunes irrelevant blocks /// according to the configured cadence. pub async fn sync_chain(&mut self) -> Result { self.ensure_genesis_in_place().await?; self.ensure_rpc_limits_in_place().await?; - let mut data = self.fetch_chain_updates().await?; + let (state_sync, mut data) = self.fetch_chain_updates().await?; // No other sync path ran, so there are no externally delivered notes to cover. - data.fetch_nullifiers(Vec::new()).await?; + state_sync.fetch_nullifiers(&mut data, Vec::new()).await?; let mut partial_mmr = self.get_current_partial_mmr().await?; - let sync_summary = self.apply_chain_updates(data, &mut partial_mmr).await?; + let sync_summary = self.apply_chain_updates(&state_sync, data, &mut partial_mmr).await?; // Cache MMR so pruning can reuse in-memory MMR. self.cache_partial_mmr(partial_mmr).await?; @@ -158,11 +158,15 @@ where /// writing anything or modifying the partial MMR. /// /// Builds the default sync input and runs [`StateSync::fetch_state`]. The nullifier check is - /// not part of this: run [`ChainSyncData::fetch_nullifiers`] on the result before applying it, - /// so it can also cover notes another sync path delivered in the same call. + /// not part of this: run [`StateSync::fetch_nullifiers`] on the result before applying it, so + /// it can also cover notes another sync path delivered in the same call. + /// + /// The [`StateSync`] is returned with the data because it must stay in scope until the update + /// is applied: its note observers accumulate per-note state during the fetch and drain it in + /// their apply hook, so [`Client::apply_chain_updates`] has to run against the same instances. /// /// Takes `&self` so it can run concurrently with the note transport sync's fetch phase. - pub async fn fetch_chain_updates(&self) -> Result { + pub async fn fetch_chain_updates(&self) -> Result<(StateSync, ChainSyncData), ClientError> { // Each `NoteObserver` owns its own per-sync state; `with_note_observer` just attaches. let note_screener = self.note_screener(); let state_sync = @@ -172,11 +176,16 @@ where let input = self.build_sync_input().await?; let block_from = block_num_from_forest(&self.get_current_partial_mmr().await?)?; - state_sync.fetch_state(block_from, input).await + let data = state_sync.fetch_state(block_from, input).await?; + + Ok((state_sync, data)) } /// Verifies fetched chain data against `partial_mmr` and writes the resulting update. /// + /// `state_sync` must be the one that produced `data`: its note observers hold the state they + /// accumulated during the fetch, and their apply hooks run here. + /// /// `partial_mmr` is loaded and cached by the caller, so one MMR can be shared with the note /// transport sync's apply phase; it is left advanced to the chain tip. /// @@ -186,6 +195,7 @@ where /// means another sync advanced the store in between and the data is stale. pub async fn apply_chain_updates( &mut self, + state_sync: &StateSync, data: ChainSyncData, partial_mmr: &mut PartialMmr, ) -> Result { @@ -197,10 +207,6 @@ where ))); } - // The observers accumulated their state during the fetch, so the hooks below must run - // against the very same instances. - let state_sync = data.state_sync.clone(); - let state_sync_update = StateSync::build_update(data, partial_mmr)?; let sync_summary: SyncSummary = (&state_sync_update).into(); @@ -251,9 +257,9 @@ where /// 1. Concurrently: the note transport fetch phase and [`Client::fetch_chain_updates`]. /// 2. The block proofs of the blocks that committed the delivered notes, which are only known /// once every transport page has been fetched. - /// 3. [`ChainSyncData::fetch_nullifiers`], covering the tracked notes *and* the ones the - /// transport just delivered, so a note delivered and consumed in the same window is reported - /// as consumed by this call. + /// 3. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the ones the transport + /// just delivered, so a note delivered and consumed in the same window is reported as + /// consumed by this call. /// 4. The writes: the transport update first, since a nullified delivered note is written by /// the chain update as an update to the row the transport insert creates. /// @@ -272,19 +278,20 @@ where self.ensure_genesis_in_place().await?; self.ensure_rpc_limits_in_place().await?; - let (mut transport_data, mut chain_data) = + let (mut transport_data, (state_sync, mut chain_data)) = futures::try_join!(self.fetch_note_transport_updates(), self.fetch_chain_updates())?; transport_data.blocks = self.fetch_note_block_proofs(&mut transport_data.imports).await?; let delivered_notes: Vec = transport_data.input_note_records().cloned().collect(); - chain_data.fetch_nullifiers(delivered_notes).await?; + state_sync.fetch_nullifiers(&mut chain_data, delivered_notes).await?; let mut partial_mmr = self.get_current_partial_mmr().await?; let new_private_notes = self.apply_note_transport_updates(transport_data, &mut partial_mmr).await?; - let mut summary = self.apply_chain_updates(chain_data, &mut partial_mmr).await?; + let mut summary = + self.apply_chain_updates(&state_sync, chain_data, &mut partial_mmr).await?; // Cache MMR so pruning can reuse in-memory MMR. self.cache_partial_mmr(partial_mmr).await?; diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index b050aaf8a4..2ac0172830 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -275,7 +275,7 @@ impl StateSync { /// /// Runs the three phases in order, each of which can also be driven separately: /// 1. [`Self::fetch_state`] — every node call but the nullifier check. - /// 2. [`ChainSyncData::fetch_nullifiers`] — the nullifier check. + /// 2. [`Self::fetch_nullifiers`] — the nullifier check. /// 3. [`Self::build_update`] — verify against the MMR and assemble the update. pub async fn sync_state( &self, @@ -285,7 +285,7 @@ impl StateSync { let block_num = block_num_from_forest(current_partial_mmr)?; let mut data = self.fetch_state(block_num, input).await?; - data.fetch_nullifiers(Vec::new()).await?; + self.fetch_nullifiers(&mut data, Vec::new()).await?; // Work on a clone so any validation failure leaves `current_partial_mmr` untouched. let mut working_mmr = current_partial_mmr.clone(); @@ -299,7 +299,7 @@ impl StateSync { /// against the client's MMR or writing anything. /// /// Runs every node call of a chain sync except the nullifier check, which - /// [`ChainSyncData::fetch_nullifiers`] performs afterwards so it can also cover notes another + /// [`Self::fetch_nullifiers`] performs afterwards so it can also cover notes another /// sync path delivered in the same call. Screening the received notes reads the store and may /// execute transactions, but nothing is persisted. /// @@ -334,7 +334,6 @@ impl StateSync { else { // No progress — already at the tip. return Ok(ChainSyncData { - state_sync: self.clone(), block_from, advance: None, note_updates, @@ -377,7 +376,6 @@ impl StateSync { self.recover_consumed_public_notes(&mut note_updates, &transactions).await?; Ok(ChainSyncData { - state_sync: self.clone(), block_from, advance: Some(ChainAdvance { chain_tip_header, @@ -455,6 +453,45 @@ impl StateSync { )) } + /// Checks the node for nullifiers of every note `data` could have consumed. + /// + /// `extra_notes` are notes another sync path fetched in the same call and is about to write — + /// the private notes delivered over the note transport layer. They are not in the store yet, + /// so they are tracked here as existing notes: that puts their nullifiers in the query and + /// lets a hit be applied to the record that will be written, which is what makes a note + /// delivered and consumed within one sync report as consumed by that same sync. + /// + /// Runs separately from [`Self::fetch_state`] so a caller syncing more than one source can + /// fetch both before checking nullifiers once, across all of them. + /// + /// No-op when the nullifier sync is disabled (see [`Self::disable_nullifier_sync`]) or when + /// the node reported no progress, since there is no block range to query. + pub async fn fetch_nullifiers( + &self, + data: &mut ChainSyncData, + extra_notes: Vec, + ) -> Result<(), ClientError> { + if !self.sync_nullifiers { + return Ok(()); + } + + let Some(chain_tip) = + data.advance.as_ref().map(|advance| advance.chain_tip_header.block_num()) + else { + return Ok(()); + }; + + data.note_updates.track_existing_input_notes(extra_notes); + + self.nullifiers_state_sync( + &mut data.note_updates, + &mut data.transaction_updates, + chain_tip, + data.block_from, + ) + .await + } + /// Recovers public notes a watched account consumed, from the `consumed_note_refs` the node /// attaches to its transactions. Fetches the body of each not-yet-tracked note by id and hands /// it to [`NoteUpdateTracker::insert_consumed_public_note`]. Notes the node doesn't return are @@ -1337,13 +1374,11 @@ impl StateSync { /// The chain data a sync fetched from the node, before any of it has been verified against the /// client's MMR or written. /// -/// Built by [`StateSync::fetch_state`], extended by [`Self::fetch_nullifiers`] and turned into a -/// [`StateSyncUpdate`] by [`StateSync::build_update`]. It carries the [`StateSync`] that produced -/// it, since the note observers accumulate per-note state during the fetch and drain it in their -/// apply hook, so the same instances have to survive into the apply phase. +/// Built by [`StateSync::fetch_state`], extended by [`StateSync::fetch_nullifiers`] and turned +/// into a [`StateSyncUpdate`] by [`StateSync::build_update`]. Carries no behavior of its own: the +/// [`StateSync`] that produced it has to stay in scope until the update is applied, because the +/// note observers accumulate per-note state during the fetch and drain it in their apply hook. pub struct ChainSyncData { - /// The component that produced this data. - pub(crate) state_sync: StateSync, /// The chain tip the sync started from. pub(crate) block_from: BlockNumber, /// What the node reported beyond `block_from`, or `None` when the client was already at the @@ -1364,44 +1399,6 @@ struct ChainAdvance { relevant_note_blocks: Vec, } -impl ChainSyncData { - /// Checks the node for nullifiers of every note this sync could have consumed. - /// - /// `extra_notes` are notes another sync path fetched in the same call and is about to write — - /// the private notes delivered over the note transport layer. They are not in the store yet, - /// so they are tracked here as existing notes: that puts their nullifiers in the query and - /// lets a hit be applied to the record that will be written, which is what makes a note - /// delivered and consumed within one sync report as consumed by that same sync. - /// - /// No-op when the nullifier sync is disabled (see [`StateSync::disable_nullifier_sync`]) or - /// when the node reported no progress, since there is no block range to query. - pub async fn fetch_nullifiers( - &mut self, - extra_notes: Vec, - ) -> Result<(), ClientError> { - if !self.state_sync.sync_nullifiers { - return Ok(()); - } - - let Some(chain_tip) = - self.advance.as_ref().map(|advance| advance.chain_tip_header.block_num()) - else { - return Ok(()); - }; - - self.note_updates.track_existing_input_notes(extra_notes); - - self.state_sync - .nullifiers_state_sync( - &mut self.note_updates, - &mut self.transaction_updates, - chain_tip, - self.block_from, - ) - .await - } -} - // HELPERS // ================================================================================================ From ce332b8400b4ab14bd0ccbf53b2462b3a4e04e33 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 27 Aug 2026 11:26:36 -0300 Subject: [PATCH 05/43] refactor(rust-client): store note blocks as they are fetched --- CHANGELOG.md | 2 +- crates/rust-client/src/note/import.rs | 70 ++++++++------------ crates/rust-client/src/note_transport/mod.rs | 37 +++-------- crates/rust-client/src/sync/block_header.rs | 35 ---------- crates/rust-client/src/sync/mod.rs | 40 ++++++----- 5 files changed, 59 insertions(+), 125 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65b7270cc1..e545183b3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ * [BREAKING][type][rust] Added the `TransactionRequestError::ForeignProcedureInputsTooLong` variant ([#2187](https://github.com/0xMiden/rust-sdk/pull/2187)). * [BREAKING][behavior][rust] `Client::sync_state` now fetches the Note Transport Layer pages and the node's chain data concurrently, and writes both afterwards, instead of running a full note transport sync before the chain sync. Notes delivered over the transport are still checked for consumption in the same call — the nullifier check runs after both fetches and covers them — but the chain sync's note-tag set is now read before they are written, so a tag first registered by this call's transport import is not part of this call's `SyncNotes` query. The transport path queries the node for those notes itself, so only other notes sharing that tag wait for the next sync. -* [BREAKING][behavior][rust] The note transport sync no longer persists its progress incrementally. The imported notes, the backfill's covered-tag set and the transport cursor are all written after every page has been fetched, so a failure part way through leaves none of them written and the next sync re-fetches. Imports dedupe by note id, so a redo is harmless. The relay outbox is unchanged: `Client::flush_relay_outbox` still persists its own remaining entries as it runs. +* [BREAKING][behavior][rust] The note transport sync no longer persists its progress incrementally. The imported notes, the backfill's covered-tag set and the transport cursor are all written after every page has been fetched, so a failure part way through leaves none of them written and the next sync re-fetches. Imports dedupe by note id, so a redo is harmless. The relay outbox and the block headers of committed notes are unaffected: both are still written as they are resolved, and both are safe to redo. ### Enhancements diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 000a36bdf8..7827dbeb22 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -12,8 +12,7 @@ use alloc::collections::{BTreeMap, BTreeSet}; use alloc::string::ToString; use alloc::vec::Vec; -use miden_protocol::block::{BlockHeader, BlockNumber}; -use miden_protocol::crypto::merkle::MerklePath; +use miden_protocol::block::BlockNumber; use miden_protocol::note::{ Note, NoteAttachments, @@ -29,7 +28,7 @@ use miden_tx::auth::TransactionAuthenticator; use crate::rpc::domain::note::{FetchedNote, ResolvedNoteContent, SyncedNote}; use crate::rpc::{NoteContentFetch, RpcError}; use crate::store::input_note_states::ExpectedNoteState; -use crate::store::{InputNoteRecord, InputNoteState, NoteFilter, StoreError}; +use crate::store::{InputNoteRecord, InputNoteState, NoteFilter}; use crate::sync::NoteTagRecord; use crate::{Client, ClientError}; @@ -181,8 +180,8 @@ where /// Each request is a note's details, the block from which its commitment should be looked for, /// and the tag to track it under. Records for notes the node has not committed are final. /// Records for committed notes come back pending, since their state transition also needs the - /// header of the block that committed them — [`Client::fetch_note_block_proofs`] fetches those - /// and finishes the records. + /// header of the block that committed them — [`Client::get_and_store_note_blocks`] + /// resolves those and finishes the records. /// /// # Errors /// @@ -273,55 +272,41 @@ where Ok(import) } - /// Fetches the header and MMR proof of every block that committed one of the notes in - /// `imports`, then finishes the records waiting on them. + /// Fetches and stores the header of every block that committed one of the notes in `imports`, + /// then finishes the records that were waiting on them. /// - /// Each block is fetched once even when it committed several notes, in one page or across - /// pages. A block the client's partial MMR already tracks has its header read from the store - /// and needs no insert, so it is absent from the returned map. - /// - /// Nothing is written and the MMR is not modified: tracking the returned headers and inserting - /// them is [`Client::apply_blocks`]'s job, so the proof paths are verified against the peaks - /// only once, next to the writes they authenticate. - pub(crate) async fn fetch_note_block_proofs( - &self, + /// Each block is resolved once even when it committed several notes, in one page or across + /// pages. A block the client's partial MMR already tracks is read from the store; the rest are + /// fetched from the node and stored with their authentication nodes. + pub(crate) async fn get_and_store_note_blocks( + &mut self, imports: &mut [ExpectedNoteImport], - ) -> Result, ClientError> { + ) -> Result<(), ClientError> { let requested_blocks: BTreeSet = imports .iter() .flat_map(|import| import.pending.iter().map(|note| note.block_num)) .collect(); if requested_blocks.is_empty() { - return Ok(BTreeMap::new()); + return Ok(()); } - let partial_mmr = self.get_current_partial_mmr().await?; + let mut partial_mmr = self.get_current_partial_mmr().await?; let mut headers = BTreeMap::new(); - let mut blocks_to_insert = BTreeMap::new(); for block_num in requested_blocks { - if partial_mmr.is_tracked(block_num.as_usize()) { - let (block_header, _) = self - .store - .get_block_header_by_num(block_num) - .await? - .ok_or(StoreError::BlockHeaderNotFound(block_num))?; - headers.insert(block_num, block_header); - continue; - } - - let (block_header, mmr_proof) = - self.rpc_api.get_block_header_with_proof(block_num).await?; - headers.insert(block_num, block_header.clone()); - blocks_to_insert.insert(block_num, (block_header, mmr_proof.merkle_path().clone())); + let block_header = + self.get_and_store_authenticated_block(block_num, &mut partial_mmr).await?; + headers.insert(block_num, block_header); } + self.cache_partial_mmr(partial_mmr).await?; + for import in imports { for mut pending in core::mem::take(&mut import.pending) { let block_header = headers .get(&pending.block_num) - .expect("every pending note's block was fetched above"); + .expect("every pending note's block was resolved above"); // `block_header_received` transitions the record's state, so it must always run. let changed = @@ -338,13 +323,14 @@ where } } - Ok(blocks_to_insert) + Ok(()) } /// Writes an [`ExpectedNoteImport`], returning the details commitments of the written records. /// - /// Block headers are not written here: [`Client::apply_blocks`] must run first so a record is - /// never persisted as committed before the header proving its inclusion. + /// Block headers are not written here: [`Client::get_and_store_note_blocks`] must run + /// first, so a record is never persisted as committed before the header proving its + /// inclusion. /// /// # Panics /// @@ -694,10 +680,10 @@ struct PendingBlockHeaderNote { /// Everything an expected-note import needs to write, with nothing written yet. /// -/// The fetch-only counterpart of the [`NoteFile::ExpectedNote`] path in -/// [`Client::import_notes`], split so every network call happens before the first write: built by -/// [`Client::fetch_expected_note_imports`] and completed by -/// [`Client::fetch_note_block_proofs`], which only read and fetch, then written by +/// The counterpart of the [`NoteFile::ExpectedNote`] path in [`Client::import_notes`], split so +/// the network work happens before the note records are written: built by +/// [`Client::fetch_expected_note_imports`], completed by +/// [`Client::get_and_store_note_blocks`], and written by /// [`Client::apply_expected_note_import`]. Used by the note transport sync, which needs its /// network phase to overlap with the chain sync's. #[derive(Default)] diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 532e0e8123..5ff51100ea 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -12,9 +12,7 @@ use core::slice; use futures::Stream; use miden_protocol::address::Address; -use miden_protocol::block::{BlockHeader, BlockNumber}; -use miden_protocol::crypto::merkle::MerklePath; -use miden_protocol::crypto::merkle::mmr::PartialMmr; +use miden_protocol::block::BlockNumber; use miden_protocol::note::{Note, NoteDetails, NoteDetailsCommitment, NoteHeader, NoteId, NoteTag}; use miden_protocol::utils::serde::Serializable; use miden_tx::auth::TransactionAuthenticator; @@ -361,13 +359,9 @@ where let mut id_by_commitment = BTreeMap::new(); let (mut import, new_cursor) = self.fetch_transport_page(cursor, ¬e_tags, &mut id_by_commitment).await?; - let blocks = self.fetch_note_block_proofs(slice::from_mut(&mut import)).await?; + self.get_and_store_note_blocks(slice::from_mut(&mut import)).await?; - let mut partial_mmr = self.get_current_partial_mmr().await?; - self.apply_blocks(blocks, &mut partial_mmr).await?; self.apply_expected_note_import(import).await?; - self.cache_partial_mmr(partial_mmr).await?; - self.store.update_note_transport_cursor(new_cursor).await?; Ok(()) @@ -513,9 +507,9 @@ where /// outbox setting and is safe to redo, so it does not affect what a failure part way through /// leaves behind for the notes, tags and cursor. /// - /// The block proofs of the notes reported as committed are *not* fetched here: they are a - /// second network pass over this result (see [`Client::fetch_note_block_proofs`]), because the - /// blocks to prove are only known once every page has been fetched. + /// The block headers of the notes the node reports as committed are not resolved here: that is + /// a second pass over this result (see [`Client::get_and_store_note_blocks`]), because the + /// blocks involved are only known once every page has been fetched. /// /// Returns empty data when note transport is not configured. pub(crate) async fn fetch_note_transport_updates( @@ -562,29 +556,23 @@ where /// the imported notes. /// /// The notes are written before the covered-tag set and the cursor, so a crash between them - /// re-fetches instead of skipping notes that were never written. `partial_mmr` is loaded and - /// cached by the caller, so one MMR can be shared with the chain sync's apply phase. - /// - /// The relay outbox is not written here: [`Client::flush_relay_outbox`] persists it itself, - /// during the fetch phase. + /// re-fetches instead of skipping notes that were never written. /// - /// [`Client::fetch_note_block_proofs`] must have run on `data` first, otherwise the records - /// waiting on a block header are still pending and the import panics. + /// Neither the relay outbox nor the block headers are written here: the outbox is persisted by + /// [`Client::flush_relay_outbox`] during the fetch, and the headers by + /// [`Client::get_and_store_note_blocks`], which must have run on `data` first. + /// Otherwise the records waiting on a block header are still pending and the import panics. pub(crate) async fn apply_note_transport_updates( &mut self, data: NoteTransportSyncData, - partial_mmr: &mut PartialMmr, ) -> Result, ClientError> { let NoteTransportSyncData { covered_tags, imports, - blocks, id_by_commitment, cursor, } = data; - self.apply_blocks(blocks, partial_mmr).await?; - let mut imported_ids = Vec::new(); for import in imports { let written = self.apply_expected_note_import(import).await?; @@ -616,7 +604,7 @@ where /// Everything the note transport sync is about to write, with nothing written yet. /// /// Built by [`Client::fetch_note_transport_updates`], completed by -/// [`Client::fetch_note_block_proofs`] and written by +/// [`Client::get_and_store_note_blocks`] and written by /// [`Client::apply_note_transport_updates`]. #[derive(Default)] pub(crate) struct NoteTransportSyncData { @@ -624,9 +612,6 @@ pub(crate) struct NoteTransportSyncData { covered_tags: Option>, /// One entry per fetched page, in fetch order. pub(crate) imports: Vec, - /// Headers of the blocks that committed the fetched notes, with the MMR proof paths the node - /// returned. Filled by [`Client::fetch_note_block_proofs`]. - pub(crate) blocks: BTreeMap, /// Note ids by details commitment, taken from the note headers the transport returned. Used /// to resolve the written records back to ids. id_by_commitment: BTreeMap, diff --git a/crates/rust-client/src/sync/block_header.rs b/crates/rust-client/src/sync/block_header.rs index 1f5d598a5e..1abddd1c61 100644 --- a/crates/rust-client/src/sync/block_header.rs +++ b/crates/rust-client/src/sync/block_header.rs @@ -1,4 +1,3 @@ -use alloc::collections::BTreeMap; use alloc::sync::Arc; use alloc::vec::Vec; @@ -118,40 +117,6 @@ impl Client { Ok(Rpo256::hash_elements(&elements)) } - /// Tracks each fetched block in `partial_mmr` and inserts its header together with the - /// authentication nodes that tracking produced. - /// - /// Tracking is what verifies the node's proof path against the current peaks, so every block - /// is tracked before the first insert: a path that doesn't verify fails with nothing written. - /// Blocks already tracked are skipped, which covers both a block from an earlier sync and the - /// same block reaching this call from two different pages. - /// - /// Loading the MMR and caching it afterwards is the caller's, so one MMR can be threaded - /// through several apply steps and cached once. - pub(crate) async fn apply_blocks( - &mut self, - blocks: BTreeMap, - partial_mmr: &mut PartialMmr, - ) -> Result<(), ClientError> { - let mut authenticated_blocks = Vec::with_capacity(blocks.len()); - for (block_num, (block_header, mmr_path)) in blocks { - if partial_mmr.is_tracked(block_num.as_usize()) { - continue; - } - - let path_nodes = - track_block_in_mmr(partial_mmr, block_num, block_header.commitment(), &mmr_path)?; - authenticated_blocks.push((block_header, path_nodes)); - } - - for (block_header, path_nodes) in authenticated_blocks { - let nodes = authenticated_block_nodes(&block_header, path_nodes); - self.store.insert_block_header(&block_header, &nodes, true).await?; - } - - Ok(()) - } - // HELPERS // -------------------------------------------------------------------------------------------- diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 9dbd8fdcdc..86bf40e952 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -238,13 +238,9 @@ where self.ensure_genesis_in_place().await?; let mut data = self.fetch_note_transport_updates().await?; - data.blocks = self.fetch_note_block_proofs(&mut data.imports).await?; + self.get_and_store_note_blocks(&mut data.imports).await?; - let mut partial_mmr = self.get_current_partial_mmr().await?; - let imported_ids = self.apply_note_transport_updates(data, &mut partial_mmr).await?; - self.cache_partial_mmr(partial_mmr).await?; - - Ok(imported_ids) + self.apply_note_transport_updates(data).await } /// Runs the full client sync: private notes from the Note Transport Layer and the client's @@ -255,23 +251,25 @@ where /// the network round trips of one sync overlap with the other's while the writes stay ordered: /// /// 1. Concurrently: the note transport fetch phase and [`Client::fetch_chain_updates`]. - /// 2. The block proofs of the blocks that committed the delivered notes, which are only known - /// once every transport page has been fetched. + /// 2. The block headers of the delivered notes the node reports as committed, which are only + /// known once every transport page has been fetched. /// 3. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the ones the transport /// just delivered, so a note delivered and consumed in the same window is reported as /// consumed by this call. - /// 4. The writes: the transport update first, since a nullified delivered note is written by - /// the chain update as an update to the row the transport insert creates. + /// 4. The note, tag and cursor writes: the transport update first, since a nullified delivered + /// note is written by the chain update as an update to the row the transport insert creates. /// - /// Fails fast on the first error: the concurrent fetch drops the other side, and neither has - /// written anything the sync depends on. The one exception is the relay outbox, which - /// [`Client::flush_relay_outbox`] persists during the fetch; a re-send that already went out - /// stays in the outbox and is retried on the next sync, which the receiver dedupes by note id. + /// Fails fast on the first error, with the note records, tags, cursor and chain update all + /// still unwritten. Two writes happen before that point, both safe to redo: the relay outbox, + /// which [`Client::flush_relay_outbox`] persists during the fetch — a re-send that already + /// went out stays in the outbox and is retried on the next sync, which the receiver dedupes by + /// note id — and the block headers from step 2, which are authenticated and idempotent to + /// insert. /// - /// Unlike the previous sequential behavior, the chain sync's input set is read before the - /// delivered notes are written, so a note tag registered by this call's transport import is - /// not part of this call's `sync_notes` query. The transport path queries the node for exactly - /// those notes itself, so only other notes sharing that tag wait for the next sync. + /// Note that the chain sync's input set is read before the delivered notes are written, so a + /// note tag registered by this call's transport import is not part of this call's `sync_notes` + /// query. The transport path queries the node for exactly those notes itself, so only other + /// notes sharing that tag wait for the next sync. pub async fn sync_state(&mut self) -> Result { // Both fetch phases need genesis in place, and connecting here means the two concurrent // futures never race on the RPC client's lazy connect. @@ -281,15 +279,15 @@ where let (mut transport_data, (state_sync, mut chain_data)) = futures::try_join!(self.fetch_note_transport_updates(), self.fetch_chain_updates())?; - transport_data.blocks = self.fetch_note_block_proofs(&mut transport_data.imports).await?; + self.get_and_store_note_blocks(&mut transport_data.imports).await?; let delivered_notes: Vec = transport_data.input_note_records().cloned().collect(); state_sync.fetch_nullifiers(&mut chain_data, delivered_notes).await?; + let new_private_notes = self.apply_note_transport_updates(transport_data).await?; + let mut partial_mmr = self.get_current_partial_mmr().await?; - let new_private_notes = - self.apply_note_transport_updates(transport_data, &mut partial_mmr).await?; let mut summary = self.apply_chain_updates(&state_sync, chain_data, &mut partial_mmr).await?; From 0ff8e1a0e7f6e831c5268a3cf6643cfc6e520902 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 27 Aug 2026 15:18:25 -0300 Subject: [PATCH 06/43] refactor(rust-client): rename the expected-note update types and fields --- crates/rust-client/src/note/import.rs | 121 +++++++++++-------- crates/rust-client/src/note/mod.rs | 2 +- crates/rust-client/src/note_transport/mod.rs | 52 ++++---- crates/rust-client/src/sync/mod.rs | 4 +- 4 files changed, 99 insertions(+), 80 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 7827dbeb22..ebea60bff5 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -186,13 +186,13 @@ where /// # Errors /// /// - If a note being imported is currently being processed by a local transaction. - pub(crate) async fn fetch_expected_note_imports( + pub(crate) async fn fetch_expected_note_updates( &self, requests: &[(NoteDetails, BlockNumber, NoteTag)], - ) -> Result { - let mut import = ExpectedNoteImport::default(); + ) -> Result { + let mut note_updates = ExpectedNoteUpdates::default(); if requests.is_empty() { - return Ok(import); + return Ok(note_updates); } // Deduplicate by details commitment, keeping the last request for each note. @@ -245,7 +245,7 @@ where let Some(SyncedNote { committed: committed_note, content }) = committed_notes_data.remove(&commitment) else { - import.notes.push(note_record); + note_updates.notes_to_write.push(note_record); continue; }; @@ -261,15 +261,15 @@ where changed |= note_record.attachments_received(attachments); } - import.pending.push(PendingBlockHeaderNote { - record: note_record, + note_updates.committed_notes_awaiting_blocks.push(CommittedNoteAwaitingBlock { + note_record, block_num: committed_note.block_num(), committed_tag: metadata.tag(), changed, }); } - Ok(import) + Ok(note_updates) } /// Fetches and stores the header of every block that committed one of the notes in `imports`, @@ -280,11 +280,13 @@ where /// fetched from the node and stored with their authentication nodes. pub(crate) async fn get_and_store_note_blocks( &mut self, - imports: &mut [ExpectedNoteImport], + note_updates: &mut [ExpectedNoteUpdates], ) -> Result<(), ClientError> { - let requested_blocks: BTreeSet = imports + let requested_blocks: BTreeSet = note_updates .iter() - .flat_map(|import| import.pending.iter().map(|note| note.block_num)) + .flat_map(|page_updates| { + page_updates.committed_notes_awaiting_blocks.iter().map(|note| note.block_num) + }) .collect(); if requested_blocks.is_empty() { @@ -302,23 +304,26 @@ where self.cache_partial_mmr(partial_mmr).await?; - for import in imports { - for mut pending in core::mem::take(&mut import.pending) { + for page_updates in note_updates { + for mut note_awaiting_block in + core::mem::take(&mut page_updates.committed_notes_awaiting_blocks) + { let block_header = headers - .get(&pending.block_num) - .expect("every pending note's block was resolved above"); + .get(¬e_awaiting_block.block_num) + .expect("every committed note's block was resolved above"); // `block_header_received` transitions the record's state, so it must always run. - let changed = - pending.changed | pending.record.block_header_received(block_header)?; + note_awaiting_block.changed |= + note_awaiting_block.note_record.block_header_received(block_header)?; - if changed { + // A record the block header left unchanged has nothing to write. + if note_awaiting_block.changed { // Once committed, the note no longer needs its expected-note tag. - import.tags_to_remove.push(NoteTagRecord::with_note_source( - pending.committed_tag, - pending.record.details_commitment(), + page_updates.tags_to_remove.push(NoteTagRecord::with_note_source( + note_awaiting_block.committed_tag, + note_awaiting_block.note_record.details_commitment(), )); - import.notes.push(pending.record); + page_updates.notes_to_write.push(note_awaiting_block.note_record); } } } @@ -326,31 +331,33 @@ where Ok(()) } - /// Writes an [`ExpectedNoteImport`], returning the details commitments of the written records. + /// Writes an [`ExpectedNoteUpdates`], returning the details commitments of the written + /// records. /// - /// Block headers are not written here: [`Client::get_and_store_note_blocks`] must run - /// first, so a record is never persisted as committed before the header proving its - /// inclusion. + /// Block headers are not written here: [`Client::get_and_store_note_blocks`] must run first, + /// so a record is never persisted as committed before the header proving its inclusion. /// /// # Panics /// - /// Panics if the import still has records waiting on a block header. - pub(crate) async fn apply_expected_note_import( + /// Panics if any committed note is still awaiting its block. + pub(crate) async fn apply_expected_note_updates( &mut self, - import: ExpectedNoteImport, + note_updates: ExpectedNoteUpdates, ) -> Result, ClientError> { assert!( - import.pending.is_empty(), - "block proofs must be fetched before an expected-note import is applied" + note_updates.committed_notes_awaiting_blocks.is_empty(), + "note blocks must be stored before the committed notes that need them" ); - for tag in import.tags_to_remove { + for tag in note_updates.tags_to_remove { self.store.remove_note_tag(tag).await?; } - let mut written = Vec::with_capacity(import.notes.len()); - for note in import.notes { + let mut written = Vec::with_capacity(note_updates.notes_to_write.len()); + for note in note_updates.notes_to_write { let details_commitment = note.details_commitment(); + // A record still expected needs its tag tracked so a later sync finds it. A committed + // one is no longer in that state, so it is skipped here. if let InputNoteState::Expected(ExpectedNoteState { tag: Some(tag), .. }) = note.state() { self.store @@ -665,41 +672,51 @@ where // EXPECTED NOTE IMPORT // ================================================================================================ -/// A record whose inclusion proof and attachments have been applied, waiting for the header of the -/// block that committed it. -struct PendingBlockHeaderNote { - /// The record, with every transition but the block header already applied. - record: InputNoteRecord, +/// An expected note the node reported as committed, with its inclusion proof and attachments +/// already applied. +/// +/// Until [`Client::get_and_store_note_blocks`] resolves the block that committed it, the record is +/// missing the block-header transition, so it cannot be written yet. +struct CommittedNoteAwaitingBlock { + /// The record. Carries every transition but the block header until the block is resolved. + note_record: InputNoteRecord, /// Block that committed the note. block_num: BlockNumber, - /// Note-source tag to drop once the record is committed. + /// Note-source tag to drop, since a committed note no longer needs to be watched for. committed_tag: NoteTag, /// Whether the inclusion-proof and attachment transitions already changed the record. changed: bool, } -/// Everything an expected-note import needs to write, with nothing written yet. +/// A batch of expected notes split by whether the node has committed them, with nothing written +/// yet. /// /// The counterpart of the [`NoteFile::ExpectedNote`] path in [`Client::import_notes`], split so /// the network work happens before the note records are written: built by -/// [`Client::fetch_expected_note_imports`], completed by +/// [`Client::fetch_expected_note_updates`], completed by /// [`Client::get_and_store_note_blocks`], and written by -/// [`Client::apply_expected_note_import`]. Used by the note transport sync, which needs its +/// [`Client::apply_expected_note_updates`]. Used by the note transport sync, which needs its /// network phase to overlap with the chain sync's. #[derive(Default)] -pub(crate) struct ExpectedNoteImport { - /// Records ready to write. - notes: Vec, - /// Records waiting on a block header. - pending: Vec, - /// Note-source tags to remove. +pub(crate) struct ExpectedNoteUpdates { + /// Records ready to write: the notes the node has not committed, plus the committed ones + /// once [`Client::get_and_store_note_blocks`] has resolved their blocks. + notes_to_write: Vec, + /// Notes the node has committed, each with the block that must be inserted before its record + /// can be. [`Client::get_and_store_note_blocks`] drains these into `notes_to_write`, dropping + /// the ones the block header leaves unchanged. + committed_notes_awaiting_blocks: Vec, + /// Note-source tags to remove, one per committed note. tags_to_remove: Vec, } -impl ExpectedNoteImport { - /// The records this import is about to write. +impl ExpectedNoteUpdates { + /// The records this batch is about to write. + /// + /// Only complete once [`Client::get_and_store_note_blocks`] has run: before that, the + /// committed notes are still awaiting their blocks and are not included. pub(crate) fn input_note_records(&self) -> impl Iterator { - self.notes.iter() + self.notes_to_write.iter() } } diff --git a/crates/rust-client/src/note/mod.rs b/crates/rust-client/src/note/mod.rs index 195a2d580c..1e002e33c2 100644 --- a/crates/rust-client/src/note/mod.rs +++ b/crates/rust-client/src/note/mod.rs @@ -69,7 +69,7 @@ use crate::store::{InputNoteRecord, NoteFilter, OutputNoteRecord}; use crate::{Client, ClientError, IdPrefixFetchError}; mod import; -pub(crate) use import::ExpectedNoteImport; +pub(crate) use import::ExpectedNoteUpdates; mod note_reader; mod note_screener; mod note_update_tracker; diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 5ff51100ea..5a45cc3605 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -25,7 +25,7 @@ use miden_tx::utils::serde::{ }; pub use self::errors::NoteTransportError; -use crate::note::ExpectedNoteImport; +use crate::note::ExpectedNoteUpdates; use crate::store::InputNoteRecord; use crate::sync::NoteTagSource; use crate::{Client, ClientError}; @@ -357,11 +357,11 @@ where let cursor = self.store.get_note_transport_cursor().await?; let mut id_by_commitment = BTreeMap::new(); - let (mut import, new_cursor) = + let (mut note_updates, new_cursor) = self.fetch_transport_page(cursor, ¬e_tags, &mut id_by_commitment).await?; - self.get_and_store_note_blocks(slice::from_mut(&mut import)).await?; + self.get_and_store_note_blocks(slice::from_mut(&mut note_updates)).await?; - self.apply_expected_note_import(import).await?; + self.apply_expected_note_updates(note_updates).await?; self.store.update_note_transport_cursor(new_cursor).await?; Ok(()) @@ -406,24 +406,25 @@ where /// Drain a single tag's full history from the transport, paging until the cursor stops /// advancing. Uses a local cursor and never touches the global one, so it cannot regress - /// steady-state progress. Returns one import per fetched page, none of them written. + /// steady-state progress. Returns one batch of updates per fetched page, none of them + /// written. async fn backfill_tag( &self, tag: NoteTag, id_by_commitment: &mut BTreeMap, - ) -> Result, ClientError> { - let mut imports = Vec::new(); + ) -> Result, ClientError> { + let mut note_updates = Vec::new(); let mut cursor = NoteTransportCursor::init(); for _ in 0..Self::MAX_BACKFILL_ITERATIONS { - let (import, new_cursor) = + let (page_updates, new_cursor) = self.fetch_transport_page(cursor, &[tag], id_by_commitment).await?; - imports.push(import); + note_updates.push(page_updates); // Terminate on any lack of forward progress. A well-behaved server returns // `new_cursor == cursor` when there are no new notes for this tag (since // `rcursor = max(cursor, max_seq_returned)`); using `<=` also handles implementations // that return an `init()` cursor on empty batches (see the in-tree mock transport). if new_cursor <= cursor { - return Ok(imports); + return Ok(note_updates); } cursor = new_cursor; } @@ -436,7 +437,7 @@ where /// Fetch one batch of notes from the note transport network for the provided tags and build /// the records they imply, without writing anything. /// - /// The server paginates; this method issues one RPC and returns the import together with the + /// The server paginates; this method issues one RPC and returns the updates together with the /// new cursor. The returned cursor equals the input cursor when the batch was empty (i.e. no /// new notes). Callers that want to drain a tag's full backlog should loop until /// `new_cursor == cursor` (see [`Client::backfill_tag`]). Callers that do steady-state polling @@ -452,7 +453,7 @@ where cursor: NoteTransportCursor, tags: &[NoteTag], id_by_commitment: &mut BTreeMap, - ) -> Result<(ExpectedNoteImport, NoteTransportCursor), ClientError> { + ) -> Result<(ExpectedNoteUpdates, NoteTransportCursor), ClientError> { // Fallback lookback window, in blocks, used only for notes the transport delivered // without a sender-provided block hint. Scanning back from sync height handles // the race where a note is committed on-chain just before the NTL delivers its data. @@ -490,9 +491,9 @@ where requests.push((NoteDetails::from(note), after_block_num, tag)); } - let import = self.fetch_expected_note_imports(&requests).await?; + let note_updates = self.fetch_expected_note_updates(&requests).await?; - Ok((import, rcursor)) + Ok((note_updates, rcursor)) } /// Fetches what the note transport sync is about to write, writing only the relay outbox. @@ -533,7 +534,8 @@ where let (mut covered, pruned, new_tags) = self.plan_backfill().await?; let backfilled = !new_tags.is_empty(); for tag in new_tags { - data.imports.extend(self.backfill_tag(tag, &mut data.id_by_commitment).await?); + data.note_updates + .extend(self.backfill_tag(tag, &mut data.id_by_commitment).await?); covered.insert(tag); } if pruned || backfilled { @@ -543,10 +545,10 @@ where let cursor = self.store.get_note_transport_cursor().await?; let note_tags: Vec = self.store.get_unique_note_tags().await?.into_iter().collect(); - let (import, new_cursor) = self + let (note_updates, new_cursor) = self .fetch_transport_page(cursor, ¬e_tags, &mut data.id_by_commitment) .await?; - data.imports.push(import); + data.note_updates.push(note_updates); data.cursor = Some(new_cursor); Ok(data) @@ -560,22 +562,22 @@ where /// /// Neither the relay outbox nor the block headers are written here: the outbox is persisted by /// [`Client::flush_relay_outbox`] during the fetch, and the headers by - /// [`Client::get_and_store_note_blocks`], which must have run on `data` first. - /// Otherwise the records waiting on a block header are still pending and the import panics. + /// [`Client::get_and_store_note_blocks`], which must have run on `data` first — otherwise + /// the committed notes are written without the block-header transition that proves them. pub(crate) async fn apply_note_transport_updates( &mut self, data: NoteTransportSyncData, ) -> Result, ClientError> { let NoteTransportSyncData { covered_tags, - imports, + note_updates, id_by_commitment, cursor, } = data; let mut imported_ids = Vec::new(); - for import in imports { - let written = self.apply_expected_note_import(import).await?; + for page_updates in note_updates { + let written = self.apply_expected_note_updates(page_updates).await?; imported_ids.extend( written .into_iter() @@ -610,8 +612,8 @@ where pub(crate) struct NoteTransportSyncData { /// Covered-tag set to persist, `None` when it did not change. covered_tags: Option>, - /// One entry per fetched page, in fetch order. - pub(crate) imports: Vec, + /// One batch per fetched page, in fetch order. + pub(crate) note_updates: Vec, /// Note ids by details commitment, taken from the note headers the transport returned. Used /// to resolve the written records back to ids. id_by_commitment: BTreeMap, @@ -625,7 +627,7 @@ impl NoteTransportSyncData { /// Used to extend the chain sync's nullifier check to the notes the transport just delivered, /// which are not in the store yet. pub(crate) fn input_note_records(&self) -> impl Iterator { - self.imports.iter().flat_map(ExpectedNoteImport::input_note_records) + self.note_updates.iter().flat_map(ExpectedNoteUpdates::input_note_records) } } diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 86bf40e952..3343e1fd41 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -238,7 +238,7 @@ where self.ensure_genesis_in_place().await?; let mut data = self.fetch_note_transport_updates().await?; - self.get_and_store_note_blocks(&mut data.imports).await?; + self.get_and_store_note_blocks(&mut data.note_updates).await?; self.apply_note_transport_updates(data).await } @@ -279,7 +279,7 @@ where let (mut transport_data, (state_sync, mut chain_data)) = futures::try_join!(self.fetch_note_transport_updates(), self.fetch_chain_updates())?; - self.get_and_store_note_blocks(&mut transport_data.imports).await?; + self.get_and_store_note_blocks(&mut transport_data.note_updates).await?; let delivered_notes: Vec = transport_data.input_note_records().cloned().collect(); From 8e886b01d127e7da10a2daa0856c3cd3d026d4ad Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 27 Aug 2026 15:23:52 -0300 Subject: [PATCH 07/43] refactor(rust-client): carry a single merged ExpectedNoteUpdates per sync --- crates/rust-client/src/note/import.rs | 64 +++++++++++--------- crates/rust-client/src/note_transport/mod.rs | 37 +++++------ 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index ebea60bff5..710cba7677 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -275,18 +275,17 @@ where /// Fetches and stores the header of every block that committed one of the notes in `imports`, /// then finishes the records that were waiting on them. /// - /// Each block is resolved once even when it committed several notes, in one page or across - /// pages. A block the client's partial MMR already tracks is read from the store; the rest are - /// fetched from the node and stored with their authentication nodes. + /// Each block is resolved once even when it committed several notes. A block the client's + /// partial MMR already tracks is read from the store; the rest are fetched from the node and + /// stored with their authentication nodes. pub(crate) async fn get_and_store_note_blocks( &mut self, - note_updates: &mut [ExpectedNoteUpdates], + note_updates: &mut ExpectedNoteUpdates, ) -> Result<(), ClientError> { let requested_blocks: BTreeSet = note_updates + .committed_notes_awaiting_blocks .iter() - .flat_map(|page_updates| { - page_updates.committed_notes_awaiting_blocks.iter().map(|note| note.block_num) - }) + .map(|note| note.block_num) .collect(); if requested_blocks.is_empty() { @@ -304,27 +303,25 @@ where self.cache_partial_mmr(partial_mmr).await?; - for page_updates in note_updates { - for mut note_awaiting_block in - core::mem::take(&mut page_updates.committed_notes_awaiting_blocks) - { - let block_header = headers - .get(¬e_awaiting_block.block_num) - .expect("every committed note's block was resolved above"); - - // `block_header_received` transitions the record's state, so it must always run. - note_awaiting_block.changed |= - note_awaiting_block.note_record.block_header_received(block_header)?; - - // A record the block header left unchanged has nothing to write. - if note_awaiting_block.changed { - // Once committed, the note no longer needs its expected-note tag. - page_updates.tags_to_remove.push(NoteTagRecord::with_note_source( - note_awaiting_block.committed_tag, - note_awaiting_block.note_record.details_commitment(), - )); - page_updates.notes_to_write.push(note_awaiting_block.note_record); - } + for mut note_awaiting_block in + core::mem::take(&mut note_updates.committed_notes_awaiting_blocks) + { + let block_header = headers + .get(¬e_awaiting_block.block_num) + .expect("every committed note's block was resolved above"); + + // `block_header_received` transitions the record's state, so it must always run. + note_awaiting_block.changed |= + note_awaiting_block.note_record.block_header_received(block_header)?; + + // A record the block header left unchanged has nothing to write. + if note_awaiting_block.changed { + // Once committed, the note no longer needs its expected-note tag. + note_updates.tags_to_remove.push(NoteTagRecord::with_note_source( + note_awaiting_block.committed_tag, + note_awaiting_block.note_record.details_commitment(), + )); + note_updates.notes_to_write.push(note_awaiting_block.note_record); } } @@ -711,6 +708,17 @@ pub(crate) struct ExpectedNoteUpdates { } impl ExpectedNoteUpdates { + /// Appends another batch to this one. + /// + /// Order is preserved, which is what makes a note returned by more than one transport page + /// resolve to the version fetched last. + pub(crate) fn merge(&mut self, other: Self) { + self.notes_to_write.extend(other.notes_to_write); + self.committed_notes_awaiting_blocks + .extend(other.committed_notes_awaiting_blocks); + self.tags_to_remove.extend(other.tags_to_remove); + } + /// The records this batch is about to write. /// /// Only complete once [`Client::get_and_store_note_blocks`] has run: before that, the diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 5a45cc3605..a8321cb5c4 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -8,7 +8,6 @@ use alloc::collections::{BTreeMap, BTreeSet}; use alloc::string::String; use alloc::sync::Arc; use alloc::vec::Vec; -use core::slice; use futures::Stream; use miden_protocol::address::Address; @@ -359,7 +358,7 @@ where let mut id_by_commitment = BTreeMap::new(); let (mut note_updates, new_cursor) = self.fetch_transport_page(cursor, ¬e_tags, &mut id_by_commitment).await?; - self.get_and_store_note_blocks(slice::from_mut(&mut note_updates)).await?; + self.get_and_store_note_blocks(&mut note_updates).await?; self.apply_expected_note_updates(note_updates).await?; self.store.update_note_transport_cursor(new_cursor).await?; @@ -406,19 +405,19 @@ where /// Drain a single tag's full history from the transport, paging until the cursor stops /// advancing. Uses a local cursor and never touches the global one, so it cannot regress - /// steady-state progress. Returns one batch of updates per fetched page, none of them - /// written. + /// steady-state progress. Returns the updates from every fetched page, merged in page order + /// and none of them written. async fn backfill_tag( &self, tag: NoteTag, id_by_commitment: &mut BTreeMap, - ) -> Result, ClientError> { - let mut note_updates = Vec::new(); + ) -> Result { + let mut note_updates = ExpectedNoteUpdates::default(); let mut cursor = NoteTransportCursor::init(); for _ in 0..Self::MAX_BACKFILL_ITERATIONS { let (page_updates, new_cursor) = self.fetch_transport_page(cursor, &[tag], id_by_commitment).await?; - note_updates.push(page_updates); + note_updates.merge(page_updates); // Terminate on any lack of forward progress. A well-behaved server returns // `new_cursor == cursor` when there are no new notes for this tag (since // `rcursor = max(cursor, max_seq_returned)`); using `<=` also handles implementations @@ -535,7 +534,7 @@ where let backfilled = !new_tags.is_empty(); for tag in new_tags { data.note_updates - .extend(self.backfill_tag(tag, &mut data.id_by_commitment).await?); + .merge(self.backfill_tag(tag, &mut data.id_by_commitment).await?); covered.insert(tag); } if pruned || backfilled { @@ -548,7 +547,7 @@ where let (note_updates, new_cursor) = self .fetch_transport_page(cursor, ¬e_tags, &mut data.id_by_commitment) .await?; - data.note_updates.push(note_updates); + data.note_updates.merge(note_updates); data.cursor = Some(new_cursor); Ok(data) @@ -575,15 +574,11 @@ where cursor, } = data; - let mut imported_ids = Vec::new(); - for page_updates in note_updates { - let written = self.apply_expected_note_updates(page_updates).await?; - imported_ids.extend( - written - .into_iter() - .filter_map(|commitment| id_by_commitment.get(&commitment).copied()), - ); - } + let written = self.apply_expected_note_updates(note_updates).await?; + let mut imported_ids: Vec = written + .into_iter() + .filter_map(|commitment| id_by_commitment.get(&commitment).copied()) + .collect(); if let Some(covered_tags) = covered_tags { self.save_covered_tags(&covered_tags).await?; @@ -612,8 +607,8 @@ where pub(crate) struct NoteTransportSyncData { /// Covered-tag set to persist, `None` when it did not change. covered_tags: Option>, - /// One batch per fetched page, in fetch order. - pub(crate) note_updates: Vec, + /// Every fetched page's updates, merged in fetch order. + pub(crate) note_updates: ExpectedNoteUpdates, /// Note ids by details commitment, taken from the note headers the transport returned. Used /// to resolve the written records back to ids. id_by_commitment: BTreeMap, @@ -627,7 +622,7 @@ impl NoteTransportSyncData { /// Used to extend the chain sync's nullifier check to the notes the transport just delivered, /// which are not in the store yet. pub(crate) fn input_note_records(&self) -> impl Iterator { - self.note_updates.iter().flat_map(ExpectedNoteUpdates::input_note_records) + self.note_updates.input_note_records() } } From 393f6b22f306cded9e567ca3d14ff953ed6e7f4f Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 27 Aug 2026 16:16:31 -0300 Subject: [PATCH 08/43] refactor(rust-client): cache and prune inside apply_chain_updates --- crates/rust-client/src/sync/mod.rs | 44 ++++++++++++------------------ 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 3343e1fd41..770dc41822 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -132,9 +132,9 @@ where /// separately. /// /// Fetches everything from the node first ([`Client::fetch_chain_updates`] and - /// [`StateSync::fetch_nullifiers`]), then applies the result - /// ([`Client::apply_chain_updates`]), caches the partial MMR, and prunes irrelevant blocks - /// according to the configured cadence. + /// [`StateSync::fetch_nullifiers`]), then applies the result with + /// [`Client::apply_chain_updates`], which also caches the partial MMR and prunes irrelevant + /// blocks according to the configured cadence. pub async fn sync_chain(&mut self) -> Result { self.ensure_genesis_in_place().await?; self.ensure_rpc_limits_in_place().await?; @@ -143,15 +143,7 @@ where // No other sync path ran, so there are no externally delivered notes to cover. state_sync.fetch_nullifiers(&mut data, Vec::new()).await?; - let mut partial_mmr = self.get_current_partial_mmr().await?; - let sync_summary = self.apply_chain_updates(&state_sync, data, &mut partial_mmr).await?; - - // Cache MMR so pruning can reuse in-memory MMR. - self.cache_partial_mmr(partial_mmr).await?; - - self.maybe_untrack_and_prune_irrelevant_blocks().await?; - - Ok(sync_summary) + self.apply_chain_updates(&state_sync, data).await } /// Fetches the node's view of everything that changed since the client's chain tip, without @@ -186,20 +178,21 @@ where /// `state_sync` must be the one that produced `data`: its note observers hold the state they /// accumulated during the fetch, and their apply hooks run here. /// - /// `partial_mmr` is loaded and cached by the caller, so one MMR can be shared with the note - /// transport sync's apply phase; it is left advanced to the chain tip. + /// Also caches the partial MMR and prunes irrelevant blocks according to the configured + /// cadence, in that order: pruning reuses the cached MMR. /// /// # Errors /// - /// Returns an error if `partial_mmr` no longer starts where the data was fetched from, which + /// Returns an error if the client no longer starts where the data was fetched from, which /// means another sync advanced the store in between and the data is stale. pub async fn apply_chain_updates( &mut self, state_sync: &StateSync, data: ChainSyncData, - partial_mmr: &mut PartialMmr, ) -> Result { - let block_from = block_num_from_forest(partial_mmr)?; + let mut partial_mmr = self.get_current_partial_mmr().await?; + + let block_from = block_num_from_forest(&partial_mmr)?; if block_from != data.block_from { return Err(ClientError::ChainValidationError(format!( "chain sync data starts at block {} but the client is at block {block_from}", @@ -207,7 +200,7 @@ where ))); } - let state_sync_update = StateSync::build_update(data, partial_mmr)?; + let state_sync_update = StateSync::build_update(data, &mut partial_mmr)?; let sync_summary: SyncSummary = (&state_sync_update).into(); debug!(sync_summary = ?sync_summary, "Sync summary computed"); @@ -224,6 +217,11 @@ where .await .map_err(ClientError::StoreError)?; + // Cache MMR so pruning can reuse in-memory MMR. + self.cache_partial_mmr(partial_mmr).await?; + + self.maybe_untrack_and_prune_irrelevant_blocks().await?; + Ok(sync_summary) } @@ -286,15 +284,7 @@ where state_sync.fetch_nullifiers(&mut chain_data, delivered_notes).await?; let new_private_notes = self.apply_note_transport_updates(transport_data).await?; - - let mut partial_mmr = self.get_current_partial_mmr().await?; - let mut summary = - self.apply_chain_updates(&state_sync, chain_data, &mut partial_mmr).await?; - - // Cache MMR so pruning can reuse in-memory MMR. - self.cache_partial_mmr(partial_mmr).await?; - - self.maybe_untrack_and_prune_irrelevant_blocks().await?; + let mut summary = self.apply_chain_updates(&state_sync, chain_data).await?; summary.new_private_notes = new_private_notes; Ok(summary) From 7888dcfee8112784f41cfa160589d799428d5d6b Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 27 Aug 2026 16:33:14 -0300 Subject: [PATCH 09/43] refactor(rust-client): fetch note blocks in the fetch phase, store them in the apply phase --- crates/rust-client/src/note/import.rs | 101 +++++++++++++------ crates/rust-client/src/note/mod.rs | 2 +- crates/rust-client/src/note_transport/mod.rs | 18 ++-- crates/rust-client/src/sync/block_header.rs | 40 ++++++++ crates/rust-client/src/sync/mod.rs | 7 +- 5 files changed, 124 insertions(+), 44 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 710cba7677..140909ccc5 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -12,7 +12,8 @@ use alloc::collections::{BTreeMap, BTreeSet}; use alloc::string::ToString; use alloc::vec::Vec; -use miden_protocol::block::BlockNumber; +use miden_protocol::block::{BlockHeader, BlockNumber}; +use miden_protocol::crypto::merkle::MerklePath; use miden_protocol::note::{ Note, NoteAttachments, @@ -28,7 +29,7 @@ use miden_tx::auth::TransactionAuthenticator; use crate::rpc::domain::note::{FetchedNote, ResolvedNoteContent, SyncedNote}; use crate::rpc::{NoteContentFetch, RpcError}; use crate::store::input_note_states::ExpectedNoteState; -use crate::store::{InputNoteRecord, InputNoteState, NoteFilter}; +use crate::store::{InputNoteRecord, InputNoteState, NoteFilter, StoreError}; use crate::sync::NoteTagRecord; use crate::{Client, ClientError}; @@ -180,7 +181,7 @@ where /// Each request is a note's details, the block from which its commitment should be looked for, /// and the tag to track it under. Records for notes the node has not committed are final. /// Records for committed notes come back pending, since their state transition also needs the - /// header of the block that committed them — [`Client::get_and_store_note_blocks`] + /// header of the block that committed them — [`Client::fetch_note_blocks`] /// resolves those and finishes the records. /// /// # Errors @@ -272,14 +273,18 @@ where Ok(note_updates) } - /// Fetches and stores the header of every block that committed one of the notes in `imports`, - /// then finishes the records that were waiting on them. + /// Fetches the header and MMR proof of every block that committed one of the notes in + /// `note_updates`, then finishes the records that were waiting on them. /// - /// Each block is resolved once even when it committed several notes. A block the client's - /// partial MMR already tracks is read from the store; the rest are fetched from the node and - /// stored with their authentication nodes. - pub(crate) async fn get_and_store_note_blocks( - &mut self, + /// Each block is fetched once even when it committed several notes. A block the client's + /// partial MMR already tracks has its header read from the store and needs no insert, so it is + /// absent from `blocks_to_insert`. + /// + /// Writes nothing and does not modify the MMR: tracking the fetched headers and storing them + /// is [`Client::insert_note_blocks`]'s job, so the proof paths are verified against the peaks + /// once, next to the writes they authenticate. + pub(crate) async fn fetch_note_blocks( + &self, note_updates: &mut ExpectedNoteUpdates, ) -> Result<(), ClientError> { let requested_blocks: BTreeSet = note_updates @@ -292,23 +297,38 @@ where return Ok(()); } - let mut partial_mmr = self.get_current_partial_mmr().await?; + let partial_mmr = self.get_current_partial_mmr().await?; - let mut headers = BTreeMap::new(); + let mut block_headers = BTreeMap::new(); for block_num in requested_blocks { - let block_header = - self.get_and_store_authenticated_block(block_num, &mut partial_mmr).await?; - headers.insert(block_num, block_header); - } + if partial_mmr.is_tracked(block_num.as_usize()) { + let (block_header, _) = self + .store + .get_block_header_by_num(block_num) + .await? + .ok_or(StoreError::BlockHeaderNotFound(block_num))?; + block_headers.insert(block_num, block_header); + continue; + } - self.cache_partial_mmr(partial_mmr).await?; + let (block_header, mmr_proof) = + self.rpc_api.get_block_header_with_proof(block_num).await?; + block_headers.insert(block_num, block_header.clone()); + note_updates.blocks_to_insert.insert( + block_num, + NoteBlockToInsert { + block_header, + mmr_path: mmr_proof.merkle_path().clone(), + }, + ); + } for mut note_awaiting_block in core::mem::take(&mut note_updates.committed_notes_awaiting_blocks) { - let block_header = headers + let block_header = block_headers .get(¬e_awaiting_block.block_num) - .expect("every committed note's block was resolved above"); + .expect("every committed note's block was fetched above"); // `block_header_received` transitions the record's state, so it must always run. note_awaiting_block.changed |= @@ -331,21 +351,28 @@ where /// Writes an [`ExpectedNoteUpdates`], returning the details commitments of the written /// records. /// - /// Block headers are not written here: [`Client::get_and_store_note_blocks`] must run first, - /// so a record is never persisted as committed before the header proving its inclusion. + /// The block headers go in first, so a record is never persisted as committed before the + /// header proving its inclusion. Caching the partial MMR is part of that, since the inserts + /// change the tracked block set. /// /// # Panics /// - /// Panics if any committed note is still awaiting its block. + /// Panics if any committed note is still awaiting its block, i.e. if + /// [`Client::fetch_note_blocks`] has not run. pub(crate) async fn apply_expected_note_updates( &mut self, note_updates: ExpectedNoteUpdates, ) -> Result, ClientError> { assert!( note_updates.committed_notes_awaiting_blocks.is_empty(), - "note blocks must be stored before the committed notes that need them" + "note blocks must be fetched before the committed notes that need them are written" ); + let mut partial_mmr = self.get_current_partial_mmr().await?; + self.insert_note_blocks(note_updates.blocks_to_insert, &mut partial_mmr).await?; + // Cache MMR so pruning can reuse in-memory MMR. + self.cache_partial_mmr(partial_mmr).await?; + for tag in note_updates.tags_to_remove { self.store.remove_note_tag(tag).await?; } @@ -672,7 +699,7 @@ where /// An expected note the node reported as committed, with its inclusion proof and attachments /// already applied. /// -/// Until [`Client::get_and_store_note_blocks`] resolves the block that committed it, the record is +/// Until [`Client::fetch_note_blocks`] resolves the block that committed it, the record is /// missing the block-header transition, so it cannot be written yet. struct CommittedNoteAwaitingBlock { /// The record. Carries every transition but the block header until the block is resolved. @@ -691,22 +718,35 @@ struct CommittedNoteAwaitingBlock { /// The counterpart of the [`NoteFile::ExpectedNote`] path in [`Client::import_notes`], split so /// the network work happens before the note records are written: built by /// [`Client::fetch_expected_note_updates`], completed by -/// [`Client::get_and_store_note_blocks`], and written by +/// [`Client::fetch_note_blocks`], and written by /// [`Client::apply_expected_note_updates`]. Used by the note transport sync, which needs its /// network phase to overlap with the chain sync's. #[derive(Default)] pub(crate) struct ExpectedNoteUpdates { /// Records ready to write: the notes the node has not committed, plus the committed ones - /// once [`Client::get_and_store_note_blocks`] has resolved their blocks. + /// once [`Client::fetch_note_blocks`] has resolved their blocks. notes_to_write: Vec, - /// Notes the node has committed, each with the block that must be inserted before its record - /// can be. [`Client::get_and_store_note_blocks`] drains these into `notes_to_write`, dropping - /// the ones the block header leaves unchanged. + /// Notes the node has committed, each waiting on the block that committed it. + /// [`Client::fetch_note_blocks`] drains these into `notes_to_write`, dropping the ones the + /// block header leaves unchanged, so this is empty by the time the batch is written. committed_notes_awaiting_blocks: Vec, + /// Blocks that must be tracked and stored before the committed notes that need them, keyed by + /// block number so a block committing several notes is stored once. Filled by + /// [`Client::fetch_note_blocks`]; blocks the client already tracks are absent. + blocks_to_insert: BTreeMap, /// Note-source tags to remove, one per committed note. tags_to_remove: Vec, } +/// A block header and the MMR proof path the node returned with it, as received. +/// +/// The path is verified against the client's peaks when the block is tracked, in +/// [`Client::insert_note_blocks`]. +pub(crate) struct NoteBlockToInsert { + pub(crate) block_header: BlockHeader, + pub(crate) mmr_path: MerklePath, +} + impl ExpectedNoteUpdates { /// Appends another batch to this one. /// @@ -716,12 +756,13 @@ impl ExpectedNoteUpdates { self.notes_to_write.extend(other.notes_to_write); self.committed_notes_awaiting_blocks .extend(other.committed_notes_awaiting_blocks); + self.blocks_to_insert.extend(other.blocks_to_insert); self.tags_to_remove.extend(other.tags_to_remove); } /// The records this batch is about to write. /// - /// Only complete once [`Client::get_and_store_note_blocks`] has run: before that, the + /// Only complete once [`Client::fetch_note_blocks`] has run: before that, the /// committed notes are still awaiting their blocks and are not included. pub(crate) fn input_note_records(&self) -> impl Iterator { self.notes_to_write.iter() diff --git a/crates/rust-client/src/note/mod.rs b/crates/rust-client/src/note/mod.rs index 1e002e33c2..8482b43676 100644 --- a/crates/rust-client/src/note/mod.rs +++ b/crates/rust-client/src/note/mod.rs @@ -69,7 +69,7 @@ use crate::store::{InputNoteRecord, NoteFilter, OutputNoteRecord}; use crate::{Client, ClientError, IdPrefixFetchError}; mod import; -pub(crate) use import::ExpectedNoteUpdates; +pub(crate) use import::{ExpectedNoteUpdates, NoteBlockToInsert}; mod note_reader; mod note_screener; mod note_update_tracker; diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index a8321cb5c4..d727d355ed 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -358,7 +358,7 @@ where let mut id_by_commitment = BTreeMap::new(); let (mut note_updates, new_cursor) = self.fetch_transport_page(cursor, ¬e_tags, &mut id_by_commitment).await?; - self.get_and_store_note_blocks(&mut note_updates).await?; + self.fetch_note_blocks(&mut note_updates).await?; self.apply_expected_note_updates(note_updates).await?; self.store.update_note_transport_cursor(new_cursor).await?; @@ -507,9 +507,9 @@ where /// outbox setting and is safe to redo, so it does not affect what a failure part way through /// leaves behind for the notes, tags and cursor. /// - /// The block headers of the notes the node reports as committed are not resolved here: that is - /// a second pass over this result (see [`Client::get_and_store_note_blocks`]), because the - /// blocks involved are only known once every page has been fetched. + /// The block headers of the notes the node reports as committed are fetched at the end, once + /// every page is in and the set of blocks involved is known. Storing them is the apply + /// phase's. /// /// Returns empty data when note transport is not configured. pub(crate) async fn fetch_note_transport_updates( @@ -550,6 +550,10 @@ where data.note_updates.merge(note_updates); data.cursor = Some(new_cursor); + // Every page is in, so the blocks that committed the delivered notes are now known. This + // finishes their records and leaves the blocks for the apply phase to store. + self.fetch_note_blocks(&mut data.note_updates).await?; + Ok(data) } @@ -559,10 +563,8 @@ where /// The notes are written before the covered-tag set and the cursor, so a crash between them /// re-fetches instead of skipping notes that were never written. /// - /// Neither the relay outbox nor the block headers are written here: the outbox is persisted by - /// [`Client::flush_relay_outbox`] during the fetch, and the headers by - /// [`Client::get_and_store_note_blocks`], which must have run on `data` first — otherwise - /// the committed notes are written without the block-header transition that proves them. + /// The relay outbox is not written here: [`Client::flush_relay_outbox`] persists it during the + /// fetch. The block headers are, ahead of the notes that need them. pub(crate) async fn apply_note_transport_updates( &mut self, data: NoteTransportSyncData, diff --git a/crates/rust-client/src/sync/block_header.rs b/crates/rust-client/src/sync/block_header.rs index 1abddd1c61..86fa437159 100644 --- a/crates/rust-client/src/sync/block_header.rs +++ b/crates/rust-client/src/sync/block_header.rs @@ -1,3 +1,4 @@ +use alloc::collections::BTreeMap; use alloc::sync::Arc; use alloc::vec::Vec; @@ -8,6 +9,7 @@ use miden_protocol::crypto::merkle::mmr::{Forest, InOrderIndex, PartialMmr}; use miden_protocol::{Felt, Word}; use tracing::warn; +use crate::note::NoteBlockToInsert; use crate::rpc::NodeRpcClient; use crate::store::{BlockRelevance, StoreError}; #[cfg(feature = "testing")] @@ -117,6 +119,44 @@ impl Client { Ok(Rpo256::hash_elements(&elements)) } + /// Tracks each fetched note block in `partial_mmr` and stores its header together with the + /// authentication nodes that tracking produced. + /// + /// Tracking is what verifies the node's proof path against the current peaks, so every block is + /// tracked before the first insert: a path that does not verify fails with nothing written. + /// Blocks already tracked are skipped, which covers a block the client picked up in an earlier + /// sync. + /// + /// The caller loads the MMR and caches it afterwards, since the inserts change the tracked + /// block set. + pub(crate) async fn insert_note_blocks( + &mut self, + blocks: BTreeMap, + partial_mmr: &mut PartialMmr, + ) -> Result<(), ClientError> { + let mut authenticated_blocks = Vec::with_capacity(blocks.len()); + for (block_num, block) in blocks { + if partial_mmr.is_tracked(block_num.as_usize()) { + continue; + } + + let path_nodes = track_block_in_mmr( + partial_mmr, + block_num, + block.block_header.commitment(), + &block.mmr_path, + )?; + authenticated_blocks.push((block.block_header, path_nodes)); + } + + for (block_header, path_nodes) in authenticated_blocks { + let nodes = authenticated_block_nodes(&block_header, path_nodes); + self.store.insert_block_header(&block_header, &nodes, true).await?; + } + + Ok(()) + } + // HELPERS // -------------------------------------------------------------------------------------------- diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 770dc41822..10fc917af3 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -235,8 +235,7 @@ where } self.ensure_genesis_in_place().await?; - let mut data = self.fetch_note_transport_updates().await?; - self.get_and_store_note_blocks(&mut data.note_updates).await?; + let data = self.fetch_note_transport_updates().await?; self.apply_note_transport_updates(data).await } @@ -274,11 +273,9 @@ where self.ensure_genesis_in_place().await?; self.ensure_rpc_limits_in_place().await?; - let (mut transport_data, (state_sync, mut chain_data)) = + let (transport_data, (state_sync, mut chain_data)) = futures::try_join!(self.fetch_note_transport_updates(), self.fetch_chain_updates())?; - self.get_and_store_note_blocks(&mut transport_data.note_updates).await?; - let delivered_notes: Vec = transport_data.input_note_records().cloned().collect(); state_sync.fetch_nullifiers(&mut chain_data, delivered_notes).await?; From 47c4bd5986e8c8923d358f088dd67bfab91ea46e Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 27 Aug 2026 17:43:04 -0300 Subject: [PATCH 10/43] docs(rust-client): correct and trim the sync phase doc comments --- crates/rust-client/src/note/import.rs | 14 +++------ crates/rust-client/src/note_transport/mod.rs | 16 ++++------ crates/rust-client/src/sync/mod.rs | 32 ++++++++------------ crates/rust-client/src/sync/state_sync.rs | 2 +- 4 files changed, 25 insertions(+), 39 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 140909ccc5..5b693bb56e 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -281,8 +281,7 @@ where /// absent from `blocks_to_insert`. /// /// Writes nothing and does not modify the MMR: tracking the fetched headers and storing them - /// is [`Client::insert_note_blocks`]'s job, so the proof paths are verified against the peaks - /// once, next to the writes they authenticate. + /// is [`Client::insert_note_blocks`]'s job. pub(crate) async fn fetch_note_blocks( &self, note_updates: &mut ExpectedNoteUpdates, @@ -352,8 +351,7 @@ where /// records. /// /// The block headers go in first, so a record is never persisted as committed before the - /// header proving its inclusion. Caching the partial MMR is part of that, since the inserts - /// change the tracked block set. + /// header proving its inclusion. /// /// # Panics /// @@ -716,11 +714,9 @@ struct CommittedNoteAwaitingBlock { /// yet. /// /// The counterpart of the [`NoteFile::ExpectedNote`] path in [`Client::import_notes`], split so -/// the network work happens before the note records are written: built by -/// [`Client::fetch_expected_note_updates`], completed by -/// [`Client::fetch_note_blocks`], and written by -/// [`Client::apply_expected_note_updates`]. Used by the note transport sync, which needs its -/// network phase to overlap with the chain sync's. +/// the network work happens before the writes: built by +/// [`Client::fetch_expected_note_updates`], completed by [`Client::fetch_note_blocks`], written by +/// [`Client::apply_expected_note_updates`]. #[derive(Default)] pub(crate) struct ExpectedNoteUpdates { /// Records ready to write: the notes the node has not committed, plus the committed ones diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index d727d355ed..684a2f3566 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -502,14 +502,10 @@ where /// [`Client::apply_note_transport_updates`]. Takes `&self` so it can run concurrently with the /// chain sync's fetch phase. /// - /// The outbox flush is the exception to this being a read-only phase: it persists its own - /// remaining entries rather than handing them to the apply phase. That write touches only the - /// outbox setting and is safe to redo, so it does not affect what a failure part way through - /// leaves behind for the notes, tags and cursor. - /// - /// The block headers of the notes the node reports as committed are fetched at the end, once - /// every page is in and the set of blocks involved is known. Storing them is the apply - /// phase's. + /// The one write it performs is the relay outbox, which [`Client::flush_relay_outbox`] + /// persists itself and which is safe to redo. Block headers for the notes the node reports as + /// committed are fetched at the end, once every page is in and the blocks involved are known; + /// storing them is the apply phase's. /// /// Returns empty data when note transport is not configured. pub(crate) async fn fetch_note_transport_updates( @@ -563,8 +559,8 @@ where /// The notes are written before the covered-tag set and the cursor, so a crash between them /// re-fetches instead of skipping notes that were never written. /// - /// The relay outbox is not written here: [`Client::flush_relay_outbox`] persists it during the - /// fetch. The block headers are, ahead of the notes that need them. + /// The block headers go in ahead of the notes that need them. The relay outbox does not: + /// [`Client::flush_relay_outbox`] persists it during the fetch. pub(crate) async fn apply_note_transport_updates( &mut self, data: NoteTransportSyncData, diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 10fc917af3..12ea1a4e80 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -173,7 +173,8 @@ where Ok((state_sync, data)) } - /// Verifies fetched chain data against `partial_mmr` and writes the resulting update. + /// Verifies fetched chain data against the client's partial MMR and writes the resulting + /// update. /// /// `state_sync` must be the one that produced `data`: its note observers hold the state they /// accumulated during the fetch, and their apply hooks run here. @@ -244,29 +245,22 @@ where /// on-chain state with the Miden node. /// /// The two are fetched concurrently, since the transport pages and the node's sync data are - /// independent. Everything that touches the MMR or the store runs sequentially afterwards, so - /// the network round trips of one sync overlap with the other's while the writes stay ordered: + /// independent, and everything that writes runs sequentially afterwards: /// - /// 1. Concurrently: the note transport fetch phase and [`Client::fetch_chain_updates`]. - /// 2. The block headers of the delivered notes the node reports as committed, which are only - /// known once every transport page has been fetched. - /// 3. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the ones the transport + /// 1. Concurrently: the note transport fetch and [`Client::fetch_chain_updates`]. + /// 2. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the ones the transport /// just delivered, so a note delivered and consumed in the same window is reported as /// consumed by this call. - /// 4. The note, tag and cursor writes: the transport update first, since a nullified delivered - /// note is written by the chain update as an update to the row the transport insert creates. + /// 3. The writes: the transport update first, since a nullified delivered note is written by + /// the chain update as an update to the row the transport insert creates. /// - /// Fails fast on the first error, with the note records, tags, cursor and chain update all - /// still unwritten. Two writes happen before that point, both safe to redo: the relay outbox, - /// which [`Client::flush_relay_outbox`] persists during the fetch — a re-send that already - /// went out stays in the outbox and is retried on the next sync, which the receiver dedupes by - /// note id — and the block headers from step 2, which are authenticated and idempotent to - /// insert. + /// Fails fast on the first error, with nothing written but the relay outbox, which + /// [`Client::flush_relay_outbox`] persists during the fetch and the next sync retries. /// - /// Note that the chain sync's input set is read before the delivered notes are written, so a - /// note tag registered by this call's transport import is not part of this call's `sync_notes` - /// query. The transport path queries the node for exactly those notes itself, so only other - /// notes sharing that tag wait for the next sync. + /// The chain sync's note tags are read before the delivered notes are written, so a note first + /// tagged by this call's transport import is not covered by this call's `sync_notes` query. If + /// its commitment falls in the range this sync advances through, the record stays expected and + /// no later query revisits that range. pub async fn sync_state(&mut self) -> Result { // Both fetch phases need genesis in place, and connecting here means the two concurrent // futures never race on the RPC client's lazy connect. diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 2ac0172830..34c7d9c9d2 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -391,7 +391,7 @@ impl StateSync { /// Verifies the fetched chain data against `partial_mmr` and turns it into the update to /// persist. /// - /// This is the only step that mutates the MMR: it applies the node's delta, checks the + /// This is the chain sync's only MMR mutation: it applies the node's delta, checks the /// resulting peaks against the chain tip header's chain commitment, and tracks the screened /// note blocks that still hold an unspent note. It performs no I/O, so every check runs before /// the caller's first write, and a failure leaves `partial_mmr` to be discarded by the caller. From 692f01b630a8c534c27c82a030b93bdf6f56d3f2 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 27 Aug 2026 23:45:51 -0300 Subject: [PATCH 11/43] fix(rust-client): screen the chain sync's notes after the transport writes --- crates/rust-client/src/sync/mod.rs | 38 +++--- crates/rust-client/src/sync/state_sync.rs | 119 ++++++++++++------ .../miden-client-tests/src/tests/transport.rs | 108 ++++++++++++++++ 3 files changed, 211 insertions(+), 54 deletions(-) diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 12ea1a4e80..25a4ec8d91 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -140,8 +140,9 @@ where self.ensure_rpc_limits_in_place().await?; let (state_sync, mut data) = self.fetch_chain_updates().await?; - // No other sync path ran, so there are no externally delivered notes to cover. - state_sync.fetch_nullifiers(&mut data, Vec::new()).await?; + // No other sync path ran, so there are no externally delivered notes to take on. + state_sync.process_fetched_state(&mut data, Vec::new()).await?; + state_sync.fetch_nullifiers(&mut data).await?; self.apply_chain_updates(&state_sync, data).await } @@ -247,20 +248,23 @@ where /// The two are fetched concurrently, since the transport pages and the node's sync data are /// independent, and everything that writes runs sequentially afterwards: /// - /// 1. Concurrently: the note transport fetch and [`Client::fetch_chain_updates`]. - /// 2. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the ones the transport - /// just delivered, so a note delivered and consumed in the same window is reported as - /// consumed by this call. - /// 3. The writes: the transport update first, since a nullified delivered note is written by - /// the chain update as an update to the row the transport insert creates. + /// 1. Concurrently: the note transport fetch and [`Client::fetch_chain_updates`]. Only node and + /// NTL calls happen here, which is all that benefits from overlapping. + /// 2. The transport writes. + /// 3. [`StateSync::process_fetched_state`], which screens the node's notes against the store — + /// hence after step 2, so a delivered note is recognised rather than discarded — and takes + /// on the delivered records so a commitment reported this sync is applied to them. + /// 4. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the delivered ones, so + /// a note delivered and consumed in the same window is reported as consumed by this call. + /// 5. The chain update, written last: a nullified delivered note is persisted as an update to + /// the row step 2 inserts. /// - /// Fails fast on the first error, with nothing written but the relay outbox, which + /// Fails fast on the first error. Before step 2 nothing is written but the relay outbox, which /// [`Client::flush_relay_outbox`] persists during the fetch and the next sync retries. /// - /// The chain sync's note tags are read before the delivered notes are written, so a note first - /// tagged by this call's transport import is not covered by this call's `sync_notes` query. If - /// its commitment falls in the range this sync advances through, the record stays expected and - /// no later query revisits that range. + /// One gap remains: the chain sync's note tags are read in step 1, so a tag *first* registered + /// by this call's transport import is not part of this call's `sync_notes` query. Notes under + /// such a tag are picked up by the next sync. pub async fn sync_state(&mut self) -> Result { // Both fetch phases need genesis in place, and connecting here means the two concurrent // futures never race on the RPC client's lazy connect. @@ -272,9 +276,15 @@ where let delivered_notes: Vec = transport_data.input_note_records().cloned().collect(); - state_sync.fetch_nullifiers(&mut chain_data, delivered_notes).await?; + // The delivered notes must be in the store before the chain data is screened: the screener + // recognises a note by looking it up there, and a private note it cannot find is discarded + // for good, since the chain's note query never revisits a block range. let new_private_notes = self.apply_note_transport_updates(transport_data).await?; + + state_sync.process_fetched_state(&mut chain_data, delivered_notes).await?; + state_sync.fetch_nullifiers(&mut chain_data).await?; + let mut summary = self.apply_chain_updates(&state_sync, chain_data).await?; summary.new_private_notes = new_private_notes; diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 34c7d9c9d2..30cd3201eb 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -275,8 +275,8 @@ impl StateSync { /// /// Runs the three phases in order, each of which can also be driven separately: /// 1. [`Self::fetch_state`] — every node call but the nullifier check. - /// 2. [`Self::fetch_nullifiers`] — the nullifier check. - /// 3. [`Self::build_update`] — verify against the MMR and assemble the update. + /// 3. [`Self::fetch_nullifiers`] — the nullifier check. + /// 4. [`Self::build_update`] — verify against the MMR and assemble the update. pub async fn sync_state( &self, current_partial_mmr: &mut PartialMmr, @@ -285,7 +285,8 @@ impl StateSync { let block_num = block_num_from_forest(current_partial_mmr)?; let mut data = self.fetch_state(block_num, input).await?; - self.fetch_nullifiers(&mut data, Vec::new()).await?; + self.process_fetched_state(&mut data, Vec::new()).await?; + self.fetch_nullifiers(&mut data).await?; // Work on a clone so any validation failure leaves `current_partial_mmr` untouched. let mut working_mmr = current_partial_mmr.clone(); @@ -298,18 +299,13 @@ impl StateSync { /// Fetches the node's view of everything that changed since `block_from`, without verifying it /// against the client's MMR or writing anything. /// - /// Runs every node call of a chain sync except the nullifier check, which - /// [`Self::fetch_nullifiers`] performs afterwards so it can also cover notes another - /// sync path delivered in the same call. Screening the received notes reads the store and may - /// execute transactions, but nothing is persisted. - /// - /// The steps are: + /// Covers the two node calls that depend on nothing but the sync input: /// 1. Fetch sync data from the node (MMR delta, note inclusions, transactions). /// 2. Update account states (fetch updated public accounts, flag mismatched private ones). - /// 3. Screen note inclusions via the configured [`OnNoteReceived`] callback. - /// 4. Process transaction inclusions (commit local txs, record external consumers, discard - /// stale/expired txs, commit output notes). - /// 5. Recover the public notes the tracked accounts consumed. + /// + /// Interpreting the response is [`Self::process_fetched_state`]'s, and the nullifier check + /// [`Self::fetch_nullifiers`]'s. Both run afterwards so a caller syncing more than one source + /// can write the other source first, and check nullifiers once across all of them. pub async fn fetch_state( &self, block_from: BlockNumber, @@ -326,7 +322,7 @@ impl StateSync { let note_tags = Arc::new(note_tags); let account_ids: Vec = accounts.iter().map(AccountHeader::id).collect(); - let mut note_updates = NoteUpdateTracker::new(input_notes, output_notes); + let note_updates = NoteUpdateTracker::new(input_notes, output_notes); let mut transaction_updates = TransactionUpdateTracker::new(uncommitted_transactions); let mut account_updates = AccountUpdates::default(); @@ -365,22 +361,14 @@ impl StateSync { transaction_updates.apply_superseded_account_state(superseded_state); } - let relevant_note_blocks = self.screen_note_blocks(note_blocks, &mut note_updates).await?; - self.apply_transactions_and_nullifiers( - &chain_tip_header, - &transactions, - &mut note_updates, - &mut transaction_updates, - )?; - - self.recover_consumed_public_notes(&mut note_updates, &transactions).await?; - Ok(ChainSyncData { block_from, advance: Some(ChainAdvance { chain_tip_header, mmr_delta, - relevant_note_blocks, + note_blocks_awaiting_screening: note_blocks, + transactions, + relevant_note_blocks: Vec::new(), }), note_updates, transaction_updates, @@ -388,6 +376,55 @@ impl StateSync { }) } + /// Turns the node's raw response into note and transaction updates. + /// + /// Screens the received notes, applies the transaction inclusions, and recovers the public + /// notes the tracked accounts consumed. Only the last of those makes a node call; the rest is + /// store reads and local execution, which is why this is split from [`Self::fetch_state`] and + /// runs afterwards rather than concurrently. + /// + /// `delivered_notes` are notes another sync path fetched in the same call and has already + /// written — the private notes delivered over the note transport layer. They are tracked here + /// so the screener's verdict has a record to apply itself to, exactly as if they had been in + /// the store when the sync input was built. + /// + /// # Ordering + /// + /// The caller must have written `delivered_notes` to the store first. + /// [`NoteScreener::on_note_received`](crate::note::NoteScreener) recognises a note by looking + /// it up in the store, and a private note it cannot find is discarded — permanently, since the + /// chain's note query never revisits a block range. + pub async fn process_fetched_state( + &self, + data: &mut ChainSyncData, + delivered_notes: Vec, + ) -> Result<(), ClientError> { + let Some(advance) = data.advance.as_mut() else { + return Ok(()); + }; + + data.note_updates.track_existing_input_notes(delivered_notes); + + advance.relevant_note_blocks = self + .screen_note_blocks( + core::mem::take(&mut advance.note_blocks_awaiting_screening), + &mut data.note_updates, + ) + .await?; + + self.apply_transactions_and_nullifiers( + &advance.chain_tip_header, + &advance.transactions, + &mut data.note_updates, + &mut data.transaction_updates, + )?; + + self.recover_consumed_public_notes(&mut data.note_updates, &advance.transactions) + .await?; + + Ok(()) + } + /// Verifies the fetched chain data against `partial_mmr` and turns it into the update to /// persist. /// @@ -413,7 +450,9 @@ impl StateSync { let Some(ChainAdvance { chain_tip_header, mmr_delta, + note_blocks_awaiting_screening, relevant_note_blocks, + .. }) = advance else { // No progress — already at the tip. @@ -425,6 +464,11 @@ impl StateSync { account_updates, )); }; + assert!( + note_blocks_awaiting_screening.is_empty(), + "note blocks must be screened before the update is built" + ); + let chain_tip = chain_tip_header.block_num(); Self::advance_mmr( @@ -455,22 +499,13 @@ impl StateSync { /// Checks the node for nullifiers of every note `data` could have consumed. /// - /// `extra_notes` are notes another sync path fetched in the same call and is about to write — - /// the private notes delivered over the note transport layer. They are not in the store yet, - /// so they are tracked here as existing notes: that puts their nullifiers in the query and - /// lets a hit be applied to the record that will be written, which is what makes a note - /// delivered and consumed within one sync report as consumed by that same sync. - /// - /// Runs separately from [`Self::fetch_state`] so a caller syncing more than one source can - /// fetch both before checking nullifiers once, across all of them. + /// The query covers every note the sync tracks, including the ones + /// [`Self::process_fetched_state`] took from another sync path — so a note delivered and + /// consumed within one sync is reported as consumed by that same sync. /// /// No-op when the nullifier sync is disabled (see [`Self::disable_nullifier_sync`]) or when /// the node reported no progress, since there is no block range to query. - pub async fn fetch_nullifiers( - &self, - data: &mut ChainSyncData, - extra_notes: Vec, - ) -> Result<(), ClientError> { + pub async fn fetch_nullifiers(&self, data: &mut ChainSyncData) -> Result<(), ClientError> { if !self.sync_nullifiers { return Ok(()); } @@ -481,8 +516,6 @@ impl StateSync { return Ok(()); }; - data.note_updates.track_existing_input_notes(extra_notes); - self.nullifiers_state_sync( &mut data.note_updates, &mut data.transaction_updates, @@ -1395,6 +1428,12 @@ struct ChainAdvance { chain_tip_header: BlockHeader, /// MMR delta from `block_from` to the chain tip, excluding the chain-tip leaf. mmr_delta: MmrDelta, + /// Note blocks as the node returned them. [`StateSync::process_fetched_state`] drains these + /// into `relevant_note_blocks`, so this is empty by the time the update is built. + note_blocks_awaiting_screening: Vec, + /// Transaction records as the node returned them, read by + /// [`StateSync::process_fetched_state`]. + transactions: Vec, /// Screened blocks holding a client-relevant note, each with its `sync_notes` MMR path. relevant_note_blocks: Vec, } diff --git a/crates/testing/miden-client-tests/src/tests/transport.rs b/crates/testing/miden-client-tests/src/tests/transport.rs index 56b54d9e6a..4ac638d618 100644 --- a/crates/testing/miden-client-tests/src/tests/transport.rs +++ b/crates/testing/miden-client-tests/src/tests/transport.rs @@ -587,6 +587,114 @@ async fn fetch_private_notes_finds_note_committed_at_sync_height() { ); } +/// A private note delivered over the NTL must be committed by the same `sync_state` call that +/// advances past its commitment block. +/// +/// The commitment is learned from two independent sources that only combine through the store: +/// the NTL supplies the note's details, which the transport half writes as an `Expected` record, +/// and the node reports the commitment for the note's tag, which the chain half screens with +/// `NoteScreener::on_note_received`. That screening is a store lookup — a private note carries no +/// details from the node, so a record it cannot find is discarded — which makes the order +/// load-bearing: the transport half must write before the chain half screens. +/// +/// This is the case the lookback in `fetch_private_notes_finds_note_committed_at_sync_height` +/// does not cover. Here the note commits *above* the client's sync height, so the transport half's +/// own commitment check (capped at the stored sync height) cannot see it and the chain half is the +/// only thing that can. The chain sync's note query is a forward-moving window, so a commitment +/// discarded here is never revisited: the record would stay `Expected` forever. +#[tokio::test] +async fn ntl_note_committed_within_the_sync_window_is_committed_by_that_sync() { + // 1. Commit a private note at block 1, then advance the chain past it. + let mut mock_chain_builder = MockChainBuilder::new(); + let mock_account = mock_chain_builder + .add_existing_mock_account(miden_testing::Auth::IncrNonce) + .unwrap(); + + let private_note = NoteBuilder::new( + mock_account.id(), + RandomCoin::new([9, 8, 7, 6].map(Felt::new_unchecked).into()), + ) + .note_type(ProtocolNoteType::Private) + .tag(NoteTag::new(0).into()) + .build() + .unwrap(); + + let spawn_note = + mock_chain_builder.add_spawn_note(std::slice::from_ref(&private_note)).unwrap(); + let mut mock_chain = mock_chain_builder.build().unwrap(); + + let tx = Box::pin( + mock_chain + .build_transaction(MockTransactionInput::AccountId(mock_account.id())) + .unauthenticated_input_note(spawn_note) + .expected_output_notes(vec![RawOutputNote::Full(private_note.clone())]) + .build() + .unwrap() + .execute(), + ) + .await + .unwrap(); + mock_chain.add_pending_executed_transaction(&tx).unwrap(); + mock_chain.prove_next_block().unwrap(); + + for _ in 0..5 { + mock_chain.prove_next_block().unwrap(); + } + + // 2. Build a client that has never synced, so its sync height sits below the note's block. + let mock_transport_node = Arc::new(RwLock::new(MockNoteTransportNode::new())); + + let rpc_api = Arc::new(MockRpcApi::new(mock_chain)); + let transport_client = MockNoteTransportApi::new(mock_transport_node.clone()); + + let mut rng = rand::rng(); + let coin_seed: [u64; 4] = rng.random(); + let rng = RandomCoin::new(coin_seed.map(|v| Felt::new_unchecked(v >> 1)).into()); + + let keystore = FilesystemKeyStore::new(temp_dir()).unwrap(); + + let builder: ClientBuilder = ClientBuilder::new() + .rpc(rpc_api) + .rng(Box::new(rng)) + .sqlite_store(create_test_store_path()) + .authenticator(Arc::new(keystore)) + .tx_discard_delta(None) + .note_transport(Arc::new(transport_client)); + + let mut client = builder.build().await.unwrap(); + client.ensure_genesis_in_place().await.unwrap(); + seed_mock_transaction_encryption_key(&mut client).await; + + client.add_note_tag(NoteTag::new(0)).await.unwrap(); + + let sync_height_before = client.get_sync_height().await.unwrap(); + assert_eq!( + sync_height_before, + BlockNumber::GENESIS, + "the note must commit above the sync height for this test to exercise the chain half" + ); + + // 3. Deliver the note over the NTL before that first sync. + let details_bytes = NoteDetails::from(private_note.clone()).to_bytes(); + mock_transport_node.write().add_note(*private_note.header(), details_bytes); + + // 4. One sync: the transport half receives the details, the chain half reports the commitment + // at block 1, and the window (genesis, tip] is consumed. + client.sync_state().await.unwrap(); + + assert!( + client.get_sync_height().await.unwrap() > BlockNumber::from(1), + "the sync must have advanced past the note's commitment block" + ); + + let committed_notes = client.get_input_notes(NoteFilter::Committed).await.unwrap(); + assert!( + committed_notes.iter().any(|note| note.id() == Some(private_note.id())), + "a delivered note committed inside the synced window must be committed by that sync; \ + leaving it expected strands it, because the chain sync never revisits that block range" + ); +} + /// A private note must reach the recipient even when the sender's first relay /// attempt fails, provided the transport later recovers. /// From 1c6d625d0416ddf6bad4c7bae2e5186484a6f60e Mon Sep 17 00:00:00 2001 From: ricomateo Date: Fri, 28 Aug 2026 17:38:34 -0300 Subject: [PATCH 12/43] fix(rust-client): check delivered notes for spends below the sync height --- crates/rust-client/src/note/import.rs | 57 ++++++++++- crates/rust-client/src/note_transport/mod.rs | 2 + .../miden-client-tests/src/tests/transport.rs | 95 +++++++++++++++++++ 3 files changed, 153 insertions(+), 1 deletion(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 5b693bb56e..701999c119 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -281,7 +281,8 @@ where /// absent from `blocks_to_insert`. /// /// Writes nothing and does not modify the MMR: tracking the fetched headers and storing them - /// is [`Client::insert_note_blocks`]'s job. + /// is [`Client::insert_note_blocks`]'s job. Whether the notes it commits have already been + /// spent is [`Client::fetch_note_nullifiers`]'s. pub(crate) async fn fetch_note_blocks( &self, note_updates: &mut ExpectedNoteUpdates, @@ -347,6 +348,60 @@ where Ok(()) } + /// Marks as consumed any note in `note_updates` that was already spent when its commitment was + /// found. + /// + /// Must run after [`Client::fetch_note_blocks`]: a note only has a nullifier once it is + /// committed, so before that there is nothing to ask about. + /// + /// The sync's own nullifier check only looks *forward*, from the sync height to the chain tip, + /// and the sync height only rises. A note committed at or below the sync height — the only + /// kind resolved here — may have been spent down there too, in a region nothing else ever + /// queries. Left unchecked it stays committed and is offered as consumable forever + /// (0xMiden/rust-sdk#2422). + /// + /// The query runs from the earliest commitment block in the batch, the tightest bound they can + /// share: a note cannot be spent before it exists. + pub(crate) async fn fetch_note_nullifiers( + &self, + note_updates: &mut ExpectedNoteUpdates, + ) -> Result<(), ClientError> { + // A record here carries a nullifier only if this batch just committed it: one that was + // already committed is dropped by `fetch_note_blocks` as unchanged. + let mut nullifiers = BTreeSet::new(); + let mut lowest_commitment_block: BlockNumber = u32::MAX.into(); + for note_record in ¬e_updates.notes_to_write { + let (Some(nullifier), Some(inclusion_proof)) = + (note_record.nullifier(), note_record.inclusion_proof()) + else { + continue; + }; + nullifiers.insert(nullifier); + lowest_commitment_block = + lowest_commitment_block.min(inclusion_proof.location().block_num()); + } + + if nullifiers.is_empty() { + return Ok(()); + } + + let spent_heights = self + .rpc_api + .get_nullifier_commit_heights(nullifiers, lowest_commitment_block) + .await?; + + for note_record in &mut note_updates.notes_to_write { + let Some(nullifier) = note_record.nullifier() else { + continue; + }; + if let Some(Some(spent_at)) = spent_heights.get(&nullifier) { + note_record.consumed_externally(nullifier, *spent_at, None)?; + } + } + + Ok(()) + } + /// Writes an [`ExpectedNoteUpdates`], returning the details commitments of the written /// records. /// diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 684a2f3566..59767ad435 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -359,6 +359,7 @@ where let (mut note_updates, new_cursor) = self.fetch_transport_page(cursor, ¬e_tags, &mut id_by_commitment).await?; self.fetch_note_blocks(&mut note_updates).await?; + self.fetch_note_nullifiers(&mut note_updates).await?; self.apply_expected_note_updates(note_updates).await?; self.store.update_note_transport_cursor(new_cursor).await?; @@ -549,6 +550,7 @@ where // Every page is in, so the blocks that committed the delivered notes are now known. This // finishes their records and leaves the blocks for the apply phase to store. self.fetch_note_blocks(&mut data.note_updates).await?; + self.fetch_note_nullifiers(&mut data.note_updates).await?; Ok(data) } diff --git a/crates/testing/miden-client-tests/src/tests/transport.rs b/crates/testing/miden-client-tests/src/tests/transport.rs index 4ac638d618..c53a46ccc3 100644 --- a/crates/testing/miden-client-tests/src/tests/transport.rs +++ b/crates/testing/miden-client-tests/src/tests/transport.rs @@ -35,6 +35,7 @@ use miden_protocol::asset::{Asset, FungibleAsset}; use miden_protocol::block::BlockNumber; use miden_protocol::crypto::rand::RandomCoin; use miden_protocol::note::NoteType as ProtocolNoteType; +use miden_protocol::testing::account_id::{ACCOUNT_ID_PRIVATE_FUNGIBLE_FAUCET, ACCOUNT_ID_SENDER}; use miden_protocol::transaction::RawOutputNote; use miden_protocol::utils::serde::Serializable; use miden_standards::note::P2idNote; @@ -695,6 +696,100 @@ async fn ntl_note_committed_within_the_sync_window_is_committed_by_that_sync() { ); } +/// A note delivered over the NTL whose nullifier is already on chain must not land consumable. +/// +/// Probe for 0xMiden/rust-sdk#2422. Commitment discovery and spend discovery have opposite time +/// orientations: the transport import looks *backwards* from the sync height for the commitment +/// (`sync_expected_notes`, plus the sender's block hint), while spend discovery only ever looks +/// *forwards*, `sync_nullifiers(prefixes, checkpoint + 1, tip)`. A note spent below the checkpoint +/// therefore imports as `Committed` and nothing later corrects it. +/// +/// The scenario is a seed restore: the transport re-serves its whole backlog to a cursor-0 client +/// whose checkpoint is already at the tip. +#[tokio::test] +async fn ntl_note_already_spent_below_the_checkpoint_is_not_left_committed() { + let sender_id: AccountId = ACCOUNT_ID_SENDER.try_into().unwrap(); + let faucet_id: AccountId = ACCOUNT_ID_PRIVATE_FUNGIBLE_FAUCET.try_into().unwrap(); + + // 1. Commit a private note to the account, then spend it — both far below the eventual tip. + let mut builder = MockChainBuilder::new(); + let account = builder.add_existing_mock_account(Auth::IncrNonce).unwrap(); + let asset = Asset::Fungible(FungibleAsset::new(faucet_id, 100u64).unwrap()); + let note = builder + .add_p2id_note(sender_id, account.id(), &[asset], ProtocolNoteType::Private) + .unwrap(); + + let mut mock_chain = builder.build().unwrap(); + mock_chain.prove_next_block().unwrap(); // block 1: the note is committed + + let consume_tx = Box::pin( + mock_chain + .build_transaction(MockTransactionInput::Account(account.clone())) + .unauthenticated_input_note(note.clone()) + .build() + .unwrap() + .execute(), + ) + .await + .unwrap(); + mock_chain.add_pending_executed_transaction(&consume_tx).unwrap(); + mock_chain.prove_next_block().unwrap(); // block 2: the nullifier is on chain + + for _ in 0..5 { + mock_chain.prove_next_block().unwrap(); + } + + // 2. A freshly restored client, tracking the note's tag, synced to the tip. + let mock_transport_node = Arc::new(RwLock::new(MockNoteTransportNode::new())); + let rpc_api = Arc::new(MockRpcApi::new(mock_chain)); + let transport_client = MockNoteTransportApi::new(mock_transport_node.clone()); + + let mut rng = rand::rng(); + let coin_seed: [u64; 4] = rng.random(); + let rng = RandomCoin::new(coin_seed.map(|v| Felt::new_unchecked(v >> 1)).into()); + let keystore = FilesystemKeyStore::new(temp_dir()).unwrap(); + + let builder: ClientBuilder = ClientBuilder::new() + .rpc(rpc_api) + .rng(Box::new(rng)) + .sqlite_store(create_test_store_path()) + .authenticator(Arc::new(keystore)) + .tx_discard_delta(None) + .note_transport(Arc::new(transport_client)); + + let mut client = builder.build().await.unwrap(); + client.ensure_genesis_in_place().await.unwrap(); + seed_mock_transaction_encryption_key(&mut client).await; + client.add_note_tag(note.metadata().tag()).await.unwrap(); + + client.sync_state().await.unwrap(); + let checkpoint = client.get_sync_height().await.unwrap(); + assert!( + checkpoint > BlockNumber::from(2), + "the spend must sit below the checkpoint for this test to exercise the gap" + ); + + // 3. The transport now re-serves the note, as it does for a cursor-0 client. + let details_bytes = NoteDetails::from(note.clone()).to_bytes(); + mock_transport_node.write().add_note(*note.header(), details_bytes); + + client.sync_state().await.unwrap(); + + // The import must have happened, otherwise the assertion below passes for the wrong reason. + let all_notes = client.get_input_notes(NoteFilter::All).await.unwrap(); + assert!( + all_notes.iter().any(|n| n.details_commitment() == note.details_commitment()), + "the delivered note should have been imported" + ); + + let committed = client.get_input_notes(NoteFilter::Committed).await.unwrap(); + assert!( + !committed.iter().any(|n| n.id() == Some(note.id())), + "a note whose nullifier is already on chain must not be imported as committed: \ + the forward-only nullifier query never revisits the block that spent it" + ); +} + /// A private note must reach the recipient even when the sender's first relay /// attempt fails, provided the transport later recovers. /// From ddd92d996322cb613cb727b77d314e3b2f563a8a Mon Sep 17 00:00:00 2001 From: ricomateo Date: Fri, 28 Aug 2026 17:45:10 -0300 Subject: [PATCH 13/43] chore: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e545183b3a..26b0f6017f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ ### Fixes +* [FIX][rust] A private note fetched from the Note Transport Layer whose nullifier is already on chain is now imported as consumed instead of committed, so `get_consumable_notes` no longer reports notes the node will reject ([#2453](https://github.com/0xMiden/rust-sdk/pull/2453)). * [FIX][rust] `ChainAnchor` deserialization no longer panics on crafted input: a partial blockchain whose tracked leaf is missing an ancestor sibling, or whose block-map key disagrees with its header, is rejected as an invalid value, and anchors tracking more blocks than a transaction can reference are rejected early with the new `ChainAnchorError::TooManyTrackedBlocks` ([#2421](https://github.com/0xMiden/rust-sdk/pull/2421)). * [FIX][rust] `Client::execute_transaction_at` now fails with the new `ChainAnchorError::AnchoredTransactionExpired` when the executed transaction's expiration block has already been reached, instead of handing back a transaction the network would reject after proving ([#2421](https://github.com/0xMiden/rust-sdk/pull/2421)). * [FIX][rust] A request that sets `ignore_invalid_input_notes` but carries no input notes, or whose notes are all screened out, no longer fails with an out-of-range note-count error from the consumption checker ([#2421](https://github.com/0xMiden/rust-sdk/pull/2421)). From a072502b1269c0d639b059ad5bfa2fe7d28c67a8 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Fri, 28 Aug 2026 18:13:07 -0300 Subject: [PATCH 14/43] refactor(rust-client): name the sync data types and bindings after what they carry --- crates/rust-client/src/note/import.rs | 24 +++---- crates/rust-client/src/note/mod.rs | 2 +- crates/rust-client/src/note_transport/mod.rs | 54 ++++++++-------- crates/rust-client/src/sync/mod.rs | 65 ++++++++++--------- crates/rust-client/src/sync/state_sync.rs | 66 +++++++++++--------- 5 files changed, 114 insertions(+), 97 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 701999c119..0964f91655 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -190,8 +190,8 @@ where pub(crate) async fn fetch_expected_note_updates( &self, requests: &[(NoteDetails, BlockNumber, NoteTag)], - ) -> Result { - let mut note_updates = ExpectedNoteUpdates::default(); + ) -> Result { + let mut note_updates = TransportNoteUpdates::default(); if requests.is_empty() { return Ok(note_updates); } @@ -285,7 +285,7 @@ where /// spent is [`Client::fetch_note_nullifiers`]'s. pub(crate) async fn fetch_note_blocks( &self, - note_updates: &mut ExpectedNoteUpdates, + note_updates: &mut TransportNoteUpdates, ) -> Result<(), ClientError> { let requested_blocks: BTreeSet = note_updates .committed_notes_awaiting_blocks @@ -364,7 +364,7 @@ where /// share: a note cannot be spent before it exists. pub(crate) async fn fetch_note_nullifiers( &self, - note_updates: &mut ExpectedNoteUpdates, + note_updates: &mut TransportNoteUpdates, ) -> Result<(), ClientError> { // A record here carries a nullifier only if this batch just committed it: one that was // already committed is dropped by `fetch_note_blocks` as unchanged. @@ -402,7 +402,7 @@ where Ok(()) } - /// Writes an [`ExpectedNoteUpdates`], returning the details commitments of the written + /// Writes a [`TransportNoteUpdates`], returning the details commitments of the written /// records. /// /// The block headers go in first, so a record is never persisted as committed before the @@ -414,7 +414,7 @@ where /// [`Client::fetch_note_blocks`] has not run. pub(crate) async fn apply_expected_note_updates( &mut self, - note_updates: ExpectedNoteUpdates, + note_updates: TransportNoteUpdates, ) -> Result, ClientError> { assert!( note_updates.committed_notes_awaiting_blocks.is_empty(), @@ -765,15 +765,15 @@ struct CommittedNoteAwaitingBlock { changed: bool, } -/// A batch of expected notes split by whether the node has committed them, with nothing written -/// yet. +/// A batch of notes fetched from the Note Transport Layer, split by whether the node has +/// committed them, with nothing written yet. /// /// The counterpart of the [`NoteFile::ExpectedNote`] path in [`Client::import_notes`], split so -/// the network work happens before the writes: built by -/// [`Client::fetch_expected_note_updates`], completed by [`Client::fetch_note_blocks`], written by +/// the network work happens before the writes: built by [`Client::fetch_expected_note_updates`], +/// completed by [`Client::fetch_note_blocks`] and [`Client::fetch_note_nullifiers`], written by /// [`Client::apply_expected_note_updates`]. #[derive(Default)] -pub(crate) struct ExpectedNoteUpdates { +pub(crate) struct TransportNoteUpdates { /// Records ready to write: the notes the node has not committed, plus the committed ones /// once [`Client::fetch_note_blocks`] has resolved their blocks. notes_to_write: Vec, @@ -798,7 +798,7 @@ pub(crate) struct NoteBlockToInsert { pub(crate) mmr_path: MerklePath, } -impl ExpectedNoteUpdates { +impl TransportNoteUpdates { /// Appends another batch to this one. /// /// Order is preserved, which is what makes a note returned by more than one transport page diff --git a/crates/rust-client/src/note/mod.rs b/crates/rust-client/src/note/mod.rs index 8482b43676..6552f0c256 100644 --- a/crates/rust-client/src/note/mod.rs +++ b/crates/rust-client/src/note/mod.rs @@ -69,7 +69,7 @@ use crate::store::{InputNoteRecord, NoteFilter, OutputNoteRecord}; use crate::{Client, ClientError, IdPrefixFetchError}; mod import; -pub(crate) use import::{ExpectedNoteUpdates, NoteBlockToInsert}; +pub(crate) use import::{NoteBlockToInsert, TransportNoteUpdates}; mod note_reader; mod note_screener; mod note_update_tracker; diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 59767ad435..3c3e5c05c5 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -24,7 +24,7 @@ use miden_tx::utils::serde::{ }; pub use self::errors::NoteTransportError; -use crate::note::ExpectedNoteUpdates; +use crate::note::TransportNoteUpdates; use crate::store::InputNoteRecord; use crate::sync::NoteTagSource; use crate::{Client, ClientError}; @@ -412,8 +412,8 @@ where &self, tag: NoteTag, id_by_commitment: &mut BTreeMap, - ) -> Result { - let mut note_updates = ExpectedNoteUpdates::default(); + ) -> Result { + let mut note_updates = TransportNoteUpdates::default(); let mut cursor = NoteTransportCursor::init(); for _ in 0..Self::MAX_BACKFILL_ITERATIONS { let (page_updates, new_cursor) = @@ -453,7 +453,7 @@ where cursor: NoteTransportCursor, tags: &[NoteTag], id_by_commitment: &mut BTreeMap, - ) -> Result<(ExpectedNoteUpdates, NoteTransportCursor), ClientError> { + ) -> Result<(TransportNoteUpdates, NoteTransportCursor), ClientError> { // Fallback lookback window, in blocks, used only for notes the transport delivered // without a sender-provided block hint. Scanning back from sync height handles // the race where a note is committed on-chain just before the NTL delivers its data. @@ -504,17 +504,20 @@ where /// chain sync's fetch phase. /// /// The one write it performs is the relay outbox, which [`Client::flush_relay_outbox`] - /// persists itself and which is safe to redo. Block headers for the notes the node reports as - /// committed are fetched at the end, once every page is in and the blocks involved are known; - /// storing them is the apply phase's. + /// persists itself and which is safe to redo. + /// + /// Two steps run at the end, once every page is in and the notes the node reports as committed + /// are known: [`Client::fetch_note_blocks`] resolves the blocks that committed them, and + /// [`Client::fetch_note_nullifiers`] checks whether any was already spent. Storing what they + /// produce is the apply phase's. /// /// Returns empty data when note transport is not configured. pub(crate) async fn fetch_note_transport_updates( &self, ) -> Result { - let mut data = NoteTransportSyncData::default(); + let mut note_transport_data = NoteTransportSyncData::default(); if !self.is_note_transport_enabled() { - return Ok(data); + return Ok(note_transport_data); } // Drain any private notes whose previous relay attempt failed. A flush error is logged, @@ -530,29 +533,30 @@ where let (mut covered, pruned, new_tags) = self.plan_backfill().await?; let backfilled = !new_tags.is_empty(); for tag in new_tags { - data.note_updates - .merge(self.backfill_tag(tag, &mut data.id_by_commitment).await?); + note_transport_data + .note_updates + .merge(self.backfill_tag(tag, &mut note_transport_data.id_by_commitment).await?); covered.insert(tag); } if pruned || backfilled { - data.covered_tags = Some(covered); + note_transport_data.covered_tags = Some(covered); } let cursor = self.store.get_note_transport_cursor().await?; let note_tags: Vec = self.store.get_unique_note_tags().await?.into_iter().collect(); let (note_updates, new_cursor) = self - .fetch_transport_page(cursor, ¬e_tags, &mut data.id_by_commitment) + .fetch_transport_page(cursor, ¬e_tags, &mut note_transport_data.id_by_commitment) .await?; - data.note_updates.merge(note_updates); - data.cursor = Some(new_cursor); + note_transport_data.note_updates.merge(note_updates); + note_transport_data.cursor = Some(new_cursor); - // Every page is in, so the blocks that committed the delivered notes are now known. This + // Every page is in, so the blocks that committed these notes are now known. This // finishes their records and leaves the blocks for the apply phase to store. - self.fetch_note_blocks(&mut data.note_updates).await?; - self.fetch_note_nullifiers(&mut data.note_updates).await?; + self.fetch_note_blocks(&mut note_transport_data.note_updates).await?; + self.fetch_note_nullifiers(&mut note_transport_data.note_updates).await?; - Ok(data) + Ok(note_transport_data) } /// Writes everything [`Client::fetch_note_transport_updates`] fetched, returning the ids of @@ -565,14 +569,14 @@ where /// [`Client::flush_relay_outbox`] persists it during the fetch. pub(crate) async fn apply_note_transport_updates( &mut self, - data: NoteTransportSyncData, + note_transport_data: NoteTransportSyncData, ) -> Result, ClientError> { let NoteTransportSyncData { covered_tags, note_updates, id_by_commitment, cursor, - } = data; + } = note_transport_data; let written = self.apply_expected_note_updates(note_updates).await?; let mut imported_ids: Vec = written @@ -600,15 +604,15 @@ where /// Everything the note transport sync is about to write, with nothing written yet. /// -/// Built by [`Client::fetch_note_transport_updates`], completed by -/// [`Client::get_and_store_note_blocks`] and written by +/// Built by [`Client::fetch_note_transport_updates`], which also completes it with +/// [`Client::fetch_note_blocks`] and [`Client::fetch_note_nullifiers`], and written by /// [`Client::apply_note_transport_updates`]. #[derive(Default)] pub(crate) struct NoteTransportSyncData { /// Covered-tag set to persist, `None` when it did not change. covered_tags: Option>, /// Every fetched page's updates, merged in fetch order. - pub(crate) note_updates: ExpectedNoteUpdates, + pub(crate) note_updates: TransportNoteUpdates, /// Note ids by details commitment, taken from the note headers the transport returned. Used /// to resolve the written records back to ids. id_by_commitment: BTreeMap, @@ -619,7 +623,7 @@ pub(crate) struct NoteTransportSyncData { impl NoteTransportSyncData { /// The records this sync is about to write. /// - /// Used to extend the chain sync's nullifier check to the notes the transport just delivered, + /// Used to extend the chain sync's nullifier check to the transport-delivered notes, /// which are not in the store yet. pub(crate) fn input_note_records(&self) -> impl Iterator { self.note_updates.input_note_records() diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 25a4ec8d91..b99349e087 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -139,12 +139,12 @@ where self.ensure_genesis_in_place().await?; self.ensure_rpc_limits_in_place().await?; - let (state_sync, mut data) = self.fetch_chain_updates().await?; - // No other sync path ran, so there are no externally delivered notes to take on. - state_sync.process_fetched_state(&mut data, Vec::new()).await?; - state_sync.fetch_nullifiers(&mut data).await?; + let (state_sync, mut chain_sync_data) = self.fetch_chain_updates().await?; + // No other sync path ran, so there are no transport-delivered notes to take on. + state_sync.process_fetched_state(&mut chain_sync_data, Vec::new()).await?; + state_sync.fetch_nullifiers(&mut chain_sync_data).await?; - self.apply_chain_updates(&state_sync, data).await + self.apply_chain_updates(&state_sync, chain_sync_data).await } /// Fetches the node's view of everything that changed since the client's chain tip, without @@ -152,7 +152,7 @@ where /// /// Builds the default sync input and runs [`StateSync::fetch_state`]. The nullifier check is /// not part of this: run [`StateSync::fetch_nullifiers`] on the result before applying it, so - /// it can also cover notes another sync path delivered in the same call. + /// it can also cover transport-delivered notes another sync path fetched in the same call. /// /// The [`StateSync`] is returned with the data because it must stay in scope until the update /// is applied: its note observers accumulate per-note state during the fetch and drain it in @@ -169,16 +169,16 @@ where let input = self.build_sync_input().await?; let block_from = block_num_from_forest(&self.get_current_partial_mmr().await?)?; - let data = state_sync.fetch_state(block_from, input).await?; + let chain_sync_data = state_sync.fetch_state(block_from, input).await?; - Ok((state_sync, data)) + Ok((state_sync, chain_sync_data)) } /// Verifies fetched chain data against the client's partial MMR and writes the resulting /// update. /// - /// `state_sync` must be the one that produced `data`: its note observers hold the state they - /// accumulated during the fetch, and their apply hooks run here. + /// `state_sync` must be the one that produced `chain_sync_data`: its note observers hold the + /// state they accumulated during the fetch, and their apply hooks run here. /// /// Also caches the partial MMR and prunes irrelevant blocks according to the configured /// cadence, in that order: pruning reuses the cached MMR. @@ -190,19 +190,19 @@ where pub async fn apply_chain_updates( &mut self, state_sync: &StateSync, - data: ChainSyncData, + chain_sync_data: ChainSyncData, ) -> Result { let mut partial_mmr = self.get_current_partial_mmr().await?; let block_from = block_num_from_forest(&partial_mmr)?; - if block_from != data.block_from { + if block_from != chain_sync_data.block_from { return Err(ClientError::ChainValidationError(format!( - "chain sync data starts at block {} but the client is at block {block_from}", - data.block_from + "chain sync chain_sync_data starts at block {} but the client is at block {block_from}", + chain_sync_data.block_from ))); } - let state_sync_update = StateSync::build_update(data, &mut partial_mmr)?; + let state_sync_update = StateSync::build_update(chain_sync_data, &mut partial_mmr)?; let sync_summary: SyncSummary = (&state_sync_update).into(); debug!(sync_summary = ?sync_summary, "Sync summary computed"); @@ -237,9 +237,9 @@ where } self.ensure_genesis_in_place().await?; - let data = self.fetch_note_transport_updates().await?; + let note_transport_data = self.fetch_note_transport_updates().await?; - self.apply_note_transport_updates(data).await + self.apply_note_transport_updates(note_transport_data).await } /// Runs the full client sync: private notes from the Note Transport Layer and the client's @@ -252,12 +252,13 @@ where /// NTL calls happen here, which is all that benefits from overlapping. /// 2. The transport writes. /// 3. [`StateSync::process_fetched_state`], which screens the node's notes against the store — - /// hence after step 2, so a delivered note is recognised rather than discarded — and takes - /// on the delivered records so a commitment reported this sync is applied to them. - /// 4. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the delivered ones, so - /// a note delivered and consumed in the same window is reported as consumed by this call. - /// 5. The chain update, written last: a nullified delivered note is persisted as an update to - /// the row step 2 inserts. + /// hence after step 2, so a transport-delivered note is recognised rather than discarded — + /// and takes on those records so a commitment reported this sync is applied to them. + /// 4. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the transport-delivered + /// ones, so a note delivered and consumed in the same window is reported as consumed by this + /// call. + /// 5. The chain update, written last: a nullified transport-delivered note is persisted as an + /// update to the row step 2 inserts. /// /// Fails fast on the first error. Before step 2 nothing is written but the relay outbox, which /// [`Client::flush_relay_outbox`] persists during the fetch and the next sync retries. @@ -271,21 +272,23 @@ where self.ensure_genesis_in_place().await?; self.ensure_rpc_limits_in_place().await?; - let (transport_data, (state_sync, mut chain_data)) = + let (note_transport_data, (state_sync, mut chain_sync_data)) = futures::try_join!(self.fetch_note_transport_updates(), self.fetch_chain_updates())?; - let delivered_notes: Vec = - transport_data.input_note_records().cloned().collect(); + let transport_delivered_notes: Vec = + note_transport_data.input_note_records().cloned().collect(); - // The delivered notes must be in the store before the chain data is screened: the screener + // These must be in the store before the chain data is screened: the screener // recognises a note by looking it up there, and a private note it cannot find is discarded // for good, since the chain's note query never revisits a block range. - let new_private_notes = self.apply_note_transport_updates(transport_data).await?; + let new_private_notes = self.apply_note_transport_updates(note_transport_data).await?; - state_sync.process_fetched_state(&mut chain_data, delivered_notes).await?; - state_sync.fetch_nullifiers(&mut chain_data).await?; + state_sync + .process_fetched_state(&mut chain_sync_data, transport_delivered_notes) + .await?; + state_sync.fetch_nullifiers(&mut chain_sync_data).await?; - let mut summary = self.apply_chain_updates(&state_sync, chain_data).await?; + let mut summary = self.apply_chain_updates(&state_sync, chain_sync_data).await?; summary.new_private_notes = new_private_notes; Ok(summary) diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 30cd3201eb..581b194e57 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -284,13 +284,13 @@ impl StateSync { ) -> Result { let block_num = block_num_from_forest(current_partial_mmr)?; - let mut data = self.fetch_state(block_num, input).await?; - self.process_fetched_state(&mut data, Vec::new()).await?; - self.fetch_nullifiers(&mut data).await?; + let mut chain_sync_data = self.fetch_state(block_num, input).await?; + self.process_fetched_state(&mut chain_sync_data, Vec::new()).await?; + self.fetch_nullifiers(&mut chain_sync_data).await?; // Work on a clone so any validation failure leaves `current_partial_mmr` untouched. let mut working_mmr = current_partial_mmr.clone(); - let update = Self::build_update(data, &mut working_mmr)?; + let update = Self::build_update(chain_sync_data, &mut working_mmr)?; *current_partial_mmr = working_mmr; Ok(update) @@ -383,44 +383,49 @@ impl StateSync { /// store reads and local execution, which is why this is split from [`Self::fetch_state`] and /// runs afterwards rather than concurrently. /// - /// `delivered_notes` are notes another sync path fetched in the same call and has already - /// written — the private notes delivered over the note transport layer. They are tracked here - /// so the screener's verdict has a record to apply itself to, exactly as if they had been in - /// the store when the sync input was built. + /// `transport_delivered_notes` are the notes another sync path fetched from the Note Transport + /// Layer in the same call and has already written. They are tracked here so the screener's + /// verdict has a record to apply itself to, exactly as if they had been in the store when the + /// sync input was built. /// /// # Ordering /// - /// The caller must have written `delivered_notes` to the store first. + /// The caller must have written `transport_delivered_notes` to the store first. /// [`NoteScreener::on_note_received`](crate::note::NoteScreener) recognises a note by looking /// it up in the store, and a private note it cannot find is discarded — permanently, since the /// chain's note query never revisits a block range. pub async fn process_fetched_state( &self, - data: &mut ChainSyncData, - delivered_notes: Vec, + chain_sync_data: &mut ChainSyncData, + transport_delivered_notes: Vec, ) -> Result<(), ClientError> { - let Some(advance) = data.advance.as_mut() else { + let Some(advance) = chain_sync_data.advance.as_mut() else { return Ok(()); }; - data.note_updates.track_existing_input_notes(delivered_notes); + chain_sync_data + .note_updates + .track_existing_input_notes(transport_delivered_notes); advance.relevant_note_blocks = self .screen_note_blocks( core::mem::take(&mut advance.note_blocks_awaiting_screening), - &mut data.note_updates, + &mut chain_sync_data.note_updates, ) .await?; self.apply_transactions_and_nullifiers( &advance.chain_tip_header, &advance.transactions, - &mut data.note_updates, - &mut data.transaction_updates, + &mut chain_sync_data.note_updates, + &mut chain_sync_data.transaction_updates, )?; - self.recover_consumed_public_notes(&mut data.note_updates, &advance.transactions) - .await?; + self.recover_consumed_public_notes( + &mut chain_sync_data.note_updates, + &advance.transactions, + ) + .await?; Ok(()) } @@ -433,7 +438,7 @@ impl StateSync { /// note blocks that still hold an unspent note. It performs no I/O, so every check runs before /// the caller's first write, and a failure leaves `partial_mmr` to be discarded by the caller. pub fn build_update( - data: ChainSyncData, + chain_sync_data: ChainSyncData, partial_mmr: &mut PartialMmr, ) -> Result { let ChainSyncData { @@ -443,7 +448,7 @@ impl StateSync { transaction_updates, account_updates, .. - } = data; + } = chain_sync_data; let mut partial_blockchain_updates = PartialBlockchainUpdates::default(); @@ -497,30 +502,35 @@ impl StateSync { )) } - /// Checks the node for nullifiers of every note `data` could have consumed. + /// Checks the node for nullifiers of every note `chain_sync_data` could have consumed. /// /// The query covers every note the sync tracks, including the ones - /// [`Self::process_fetched_state`] took from another sync path — so a note delivered and + /// [`Self::process_fetched_state`] took from another sync path — so a transport-delivered note /// consumed within one sync is reported as consumed by that same sync. /// /// No-op when the nullifier sync is disabled (see [`Self::disable_nullifier_sync`]) or when /// the node reported no progress, since there is no block range to query. - pub async fn fetch_nullifiers(&self, data: &mut ChainSyncData) -> Result<(), ClientError> { + pub async fn fetch_nullifiers( + &self, + chain_sync_data: &mut ChainSyncData, + ) -> Result<(), ClientError> { if !self.sync_nullifiers { return Ok(()); } - let Some(chain_tip) = - data.advance.as_ref().map(|advance| advance.chain_tip_header.block_num()) + let Some(chain_tip) = chain_sync_data + .advance + .as_ref() + .map(|advance| advance.chain_tip_header.block_num()) else { return Ok(()); }; self.nullifiers_state_sync( - &mut data.note_updates, - &mut data.transaction_updates, + &mut chain_sync_data.note_updates, + &mut chain_sync_data.transaction_updates, chain_tip, - data.block_from, + chain_sync_data.block_from, ) .await } From f6642f3f6800466fb3cfea1c9958111d91bdc21f Mon Sep 17 00:00:00 2001 From: ricomateo Date: Fri, 28 Aug 2026 18:41:35 -0300 Subject: [PATCH 15/43] refactor(rust-client): name the transport fetch fns after what they return --- crates/rust-client/src/note/import.rs | 17 ++++++++------- crates/rust-client/src/note_transport/mod.rs | 23 ++++++++++++-------- crates/rust-client/src/sync/mod.rs | 4 ++-- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 0964f91655..f63f31106d 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -173,21 +173,22 @@ where Ok(imported_commitments) } - // FETCH-ONLY EXPECTED NOTE IMPORT + // TRANSPORT-DELIVERED NOTE IMPORT // -------------------------------------------------------------------------------------------- - /// Builds the records for a batch of expected notes without writing anything. + /// Asks the node whether a batch of transport-delivered notes is already on chain, and builds + /// their records, without writing anything. /// /// Each request is a note's details, the block from which its commitment should be looked for, /// and the tag to track it under. Records for notes the node has not committed are final. /// Records for committed notes come back pending, since their state transition also needs the - /// header of the block that committed them — [`Client::fetch_note_blocks`] - /// resolves those and finishes the records. + /// header of the block that committed them — [`Client::fetch_note_blocks`] resolves those and + /// finishes the records. /// /// # Errors /// /// - If a note being imported is currently being processed by a local transaction. - pub(crate) async fn fetch_expected_note_updates( + pub(crate) async fn fetch_transport_notes_onchain_state( &self, requests: &[(NoteDetails, BlockNumber, NoteTag)], ) -> Result { @@ -769,9 +770,9 @@ struct CommittedNoteAwaitingBlock { /// committed them, with nothing written yet. /// /// The counterpart of the [`NoteFile::ExpectedNote`] path in [`Client::import_notes`], split so -/// the network work happens before the writes: built by [`Client::fetch_expected_note_updates`], -/// completed by [`Client::fetch_note_blocks`] and [`Client::fetch_note_nullifiers`], written by -/// [`Client::apply_expected_note_updates`]. +/// the network work happens before the writes: built by +/// [`Client::fetch_transport_notes_onchain_state`], completed by [`Client::fetch_note_blocks`] and +/// [`Client::fetch_note_nullifiers`], written by [`Client::apply_expected_note_updates`]. #[derive(Default)] pub(crate) struct TransportNoteUpdates { /// Records ready to write: the notes the node has not committed, plus the committed ones diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 3c3e5c05c5..14ade917a2 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -356,8 +356,9 @@ where let cursor = self.store.get_note_transport_cursor().await?; let mut id_by_commitment = BTreeMap::new(); - let (mut note_updates, new_cursor) = - self.fetch_transport_page(cursor, ¬e_tags, &mut id_by_commitment).await?; + let (mut note_updates, new_cursor) = self + .fetch_note_transport_updates(cursor, ¬e_tags, &mut id_by_commitment) + .await?; self.fetch_note_blocks(&mut note_updates).await?; self.fetch_note_nullifiers(&mut note_updates).await?; @@ -417,7 +418,7 @@ where let mut cursor = NoteTransportCursor::init(); for _ in 0..Self::MAX_BACKFILL_ITERATIONS { let (page_updates, new_cursor) = - self.fetch_transport_page(cursor, &[tag], id_by_commitment).await?; + self.fetch_note_transport_updates(cursor, &[tag], id_by_commitment).await?; note_updates.merge(page_updates); // Terminate on any lack of forward progress. A well-behaved server returns // `new_cursor == cursor` when there are no new notes for this tag (since @@ -448,7 +449,7 @@ where /// written records back to note ids once the final record set is known. Persistence of the /// returned cursor is left to the caller so that drain loops can guard against regression of /// an already-advanced stored cursor. - async fn fetch_transport_page( + async fn fetch_note_transport_updates( &self, cursor: NoteTransportCursor, tags: &[NoteTag], @@ -491,7 +492,7 @@ where requests.push((NoteDetails::from(note), after_block_num, tag)); } - let note_updates = self.fetch_expected_note_updates(&requests).await?; + let note_updates = self.fetch_transport_notes_onchain_state(&requests).await?; Ok((note_updates, rcursor)) } @@ -512,7 +513,7 @@ where /// produce is the apply phase's. /// /// Returns empty data when note transport is not configured. - pub(crate) async fn fetch_note_transport_updates( + pub(crate) async fn fetch_note_transport_sync_data( &self, ) -> Result { let mut note_transport_data = NoteTransportSyncData::default(); @@ -546,7 +547,11 @@ where let note_tags: Vec = self.store.get_unique_note_tags().await?.into_iter().collect(); let (note_updates, new_cursor) = self - .fetch_transport_page(cursor, ¬e_tags, &mut note_transport_data.id_by_commitment) + .fetch_note_transport_updates( + cursor, + ¬e_tags, + &mut note_transport_data.id_by_commitment, + ) .await?; note_transport_data.note_updates.merge(note_updates); note_transport_data.cursor = Some(new_cursor); @@ -559,7 +564,7 @@ where Ok(note_transport_data) } - /// Writes everything [`Client::fetch_note_transport_updates`] fetched, returning the ids of + /// Writes everything [`Client::fetch_note_transport_sync_data`] fetched, returning the ids of /// the imported notes. /// /// The notes are written before the covered-tag set and the cursor, so a crash between them @@ -604,7 +609,7 @@ where /// Everything the note transport sync is about to write, with nothing written yet. /// -/// Built by [`Client::fetch_note_transport_updates`], which also completes it with +/// Built by [`Client::fetch_note_transport_sync_data`], which also completes it with /// [`Client::fetch_note_blocks`] and [`Client::fetch_note_nullifiers`], and written by /// [`Client::apply_note_transport_updates`]. #[derive(Default)] diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index b99349e087..f07fbe6013 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -237,7 +237,7 @@ where } self.ensure_genesis_in_place().await?; - let note_transport_data = self.fetch_note_transport_updates().await?; + let note_transport_data = self.fetch_note_transport_sync_data().await?; self.apply_note_transport_updates(note_transport_data).await } @@ -273,7 +273,7 @@ where self.ensure_rpc_limits_in_place().await?; let (note_transport_data, (state_sync, mut chain_sync_data)) = - futures::try_join!(self.fetch_note_transport_updates(), self.fetch_chain_updates())?; + futures::try_join!(self.fetch_note_transport_sync_data(), self.fetch_chain_updates())?; let transport_delivered_notes: Vec = note_transport_data.input_note_records().cloned().collect(); From f1d251c9ee4a843b20dfb10e33147fceb3d10c16 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Fri, 28 Aug 2026 18:44:08 -0300 Subject: [PATCH 16/43] refactor(rust-client): pair the transport fetch and apply fns by the type they carry --- crates/rust-client/src/note/import.rs | 4 ++-- crates/rust-client/src/note_transport/mod.rs | 12 ++++++------ crates/rust-client/src/sync/mod.rs | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index f63f31106d..eded0ada9d 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -413,7 +413,7 @@ where /// /// Panics if any committed note is still awaiting its block, i.e. if /// [`Client::fetch_note_blocks`] has not run. - pub(crate) async fn apply_expected_note_updates( + pub(crate) async fn apply_note_transport_updates( &mut self, note_updates: TransportNoteUpdates, ) -> Result, ClientError> { @@ -772,7 +772,7 @@ struct CommittedNoteAwaitingBlock { /// The counterpart of the [`NoteFile::ExpectedNote`] path in [`Client::import_notes`], split so /// the network work happens before the writes: built by /// [`Client::fetch_transport_notes_onchain_state`], completed by [`Client::fetch_note_blocks`] and -/// [`Client::fetch_note_nullifiers`], written by [`Client::apply_expected_note_updates`]. +/// [`Client::fetch_note_nullifiers`], written by [`Client::apply_note_transport_updates`]. #[derive(Default)] pub(crate) struct TransportNoteUpdates { /// Records ready to write: the notes the node has not committed, plus the committed ones diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 14ade917a2..8e0a137f14 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -362,7 +362,7 @@ where self.fetch_note_blocks(&mut note_updates).await?; self.fetch_note_nullifiers(&mut note_updates).await?; - self.apply_expected_note_updates(note_updates).await?; + self.apply_note_transport_updates(note_updates).await?; self.store.update_note_transport_cursor(new_cursor).await?; Ok(()) @@ -501,8 +501,8 @@ where /// /// Runs the relay-outbox flush, the per-tag history backfill and the steady-state page, and /// returns the latter two as a [`NoteTransportSyncData`] for - /// [`Client::apply_note_transport_updates`]. Takes `&self` so it can run concurrently with the - /// chain sync's fetch phase. + /// [`Client::apply_note_transport_sync_data`]. Takes `&self` so it can run concurrently with + /// the chain sync's fetch phase. /// /// The one write it performs is the relay outbox, which [`Client::flush_relay_outbox`] /// persists itself and which is safe to redo. @@ -572,7 +572,7 @@ where /// /// The block headers go in ahead of the notes that need them. The relay outbox does not: /// [`Client::flush_relay_outbox`] persists it during the fetch. - pub(crate) async fn apply_note_transport_updates( + pub(crate) async fn apply_note_transport_sync_data( &mut self, note_transport_data: NoteTransportSyncData, ) -> Result, ClientError> { @@ -583,7 +583,7 @@ where cursor, } = note_transport_data; - let written = self.apply_expected_note_updates(note_updates).await?; + let written = self.apply_note_transport_updates(note_updates).await?; let mut imported_ids: Vec = written .into_iter() .filter_map(|commitment| id_by_commitment.get(&commitment).copied()) @@ -611,7 +611,7 @@ where /// /// Built by [`Client::fetch_note_transport_sync_data`], which also completes it with /// [`Client::fetch_note_blocks`] and [`Client::fetch_note_nullifiers`], and written by -/// [`Client::apply_note_transport_updates`]. +/// [`Client::apply_note_transport_sync_data`]. #[derive(Default)] pub(crate) struct NoteTransportSyncData { /// Covered-tag set to persist, `None` when it did not change. diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index f07fbe6013..ab8eaaf57d 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -239,7 +239,7 @@ where let note_transport_data = self.fetch_note_transport_sync_data().await?; - self.apply_note_transport_updates(note_transport_data).await + self.apply_note_transport_sync_data(note_transport_data).await } /// Runs the full client sync: private notes from the Note Transport Layer and the client's @@ -281,7 +281,7 @@ where // These must be in the store before the chain data is screened: the screener // recognises a note by looking it up there, and a private note it cannot find is discarded // for good, since the chain's note query never revisits a block range. - let new_private_notes = self.apply_note_transport_updates(note_transport_data).await?; + let new_private_notes = self.apply_note_transport_sync_data(note_transport_data).await?; state_sync .process_fetched_state(&mut chain_sync_data, transport_delivered_notes) From 2669f293a923d464288edb8c3c25a4c3dd78efb9 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Fri, 28 Aug 2026 19:08:29 -0300 Subject: [PATCH 17/43] chore: simplify doc comments --- crates/rust-client/src/note/import.rs | 40 +++++++++------------------ 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index eded0ada9d..7b3b529cea 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -176,14 +176,13 @@ where // TRANSPORT-DELIVERED NOTE IMPORT // -------------------------------------------------------------------------------------------- - /// Asks the node whether a batch of transport-delivered notes is already on chain, and builds - /// their records, without writing anything. + /// Fetches the state of transport-delivered notes from the RPC, and returns a + /// [`TransportNoteUpdates`] containing the expected notes that are ready to write to the + /// store, along with the committed notes that wait for [`Client::fetch_note_blocks`] to + /// resolve their blocks. /// - /// Each request is a note's details, the block from which its commitment should be looked for, - /// and the tag to track it under. Records for notes the node has not committed are final. - /// Records for committed notes come back pending, since their state transition also needs the - /// header of the block that committed them — [`Client::fetch_note_blocks`] resolves those and - /// finishes the records. + /// The `requests` parameter contains, for each transport-delivered note, the note details, + /// the block from which its commitment should be looked for, and the tag to track it under. /// /// # Errors /// @@ -349,20 +348,11 @@ where Ok(()) } - /// Marks as consumed any note in `note_updates` that was already spent when its commitment was - /// found. + /// Fetches the nullifier commit heights of the notes in `note_updates`, marking those already + /// spent as consumed. /// /// Must run after [`Client::fetch_note_blocks`]: a note only has a nullifier once it is /// committed, so before that there is nothing to ask about. - /// - /// The sync's own nullifier check only looks *forward*, from the sync height to the chain tip, - /// and the sync height only rises. A note committed at or below the sync height — the only - /// kind resolved here — may have been spent down there too, in a region nothing else ever - /// queries. Left unchecked it stays committed and is offered as consumable forever - /// (0xMiden/rust-sdk#2422). - /// - /// The query runs from the earliest commitment block in the batch, the tightest bound they can - /// share: a note cannot be spent before it exists. pub(crate) async fn fetch_note_nullifiers( &self, note_updates: &mut TransportNoteUpdates, @@ -403,8 +393,8 @@ where Ok(()) } - /// Writes a [`TransportNoteUpdates`], returning the details commitments of the written - /// records. + /// Applies the changes from `note_updates` to the store, returning the details commitments + /// of the written records. /// /// The block headers go in first, so a record is never persisted as committed before the /// header proving its inclusion. @@ -766,13 +756,9 @@ struct CommittedNoteAwaitingBlock { changed: bool, } -/// A batch of notes fetched from the Note Transport Layer, split by whether the node has -/// committed them, with nothing written yet. -/// -/// The counterpart of the [`NoteFile::ExpectedNote`] path in [`Client::import_notes`], split so -/// the network work happens before the writes: built by -/// [`Client::fetch_transport_notes_onchain_state`], completed by [`Client::fetch_note_blocks`] and -/// [`Client::fetch_note_nullifiers`], written by [`Client::apply_note_transport_updates`]. +/// Notes fetched from the Note Transport Layer, split by whether the node has +/// committed them, along with note tags to remove, and the blocks that must be +/// stored before their corresponding committed notes. #[derive(Default)] pub(crate) struct TransportNoteUpdates { /// Records ready to write: the notes the node has not committed, plus the committed ones From c79d60acde02670aa4d01e8db462e8670a109964 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Fri, 28 Aug 2026 19:37:04 -0300 Subject: [PATCH 18/43] chore: improve doc comments --- crates/rust-client/src/note/import.rs | 13 ++----------- .../src/note/note_update_tracker.rs | 2 +- crates/rust-client/src/sync/block_header.rs | 8 -------- crates/rust-client/src/sync/mod.rs | 18 ++++-------------- .../miden-client-tests/src/tests/transport.rs | 11 ++--------- 5 files changed, 9 insertions(+), 43 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 7b3b529cea..6b1ce497f5 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -397,21 +397,12 @@ where /// of the written records. /// /// The block headers go in first, so a record is never persisted as committed before the - /// header proving its inclusion. - /// - /// # Panics - /// - /// Panics if any committed note is still awaiting its block, i.e. if - /// [`Client::fetch_note_blocks`] has not run. + /// header proving its inclusion. [`Client::fetch_note_blocks`] must have run beforehand: a + /// note still awaiting its block is not written. pub(crate) async fn apply_note_transport_updates( &mut self, note_updates: TransportNoteUpdates, ) -> Result, ClientError> { - assert!( - note_updates.committed_notes_awaiting_blocks.is_empty(), - "note blocks must be fetched before the committed notes that need them are written" - ); - let mut partial_mmr = self.get_current_partial_mmr().await?; self.insert_note_blocks(note_updates.blocks_to_insert, &mut partial_mmr).await?; // Cache MMR so pruning can reuse in-memory MMR. diff --git a/crates/rust-client/src/note/note_update_tracker.rs b/crates/rust-client/src/note/note_update_tracker.rs index ab999d7829..710caac687 100644 --- a/crates/rust-client/src/note/note_update_tracker.rs +++ b/crates/rust-client/src/note/note_update_tracker.rs @@ -377,7 +377,7 @@ impl NoteUpdateTracker { }) } - /// Tracks additional already-persisted input notes as unmodified context. + /// Tracks additional already-persisted input notes. /// /// Used to extend a sync's nullifier check to notes that are about to be written by another /// path (e.g. the note transport sync) and are therefore absent from the store snapshot this diff --git a/crates/rust-client/src/sync/block_header.rs b/crates/rust-client/src/sync/block_header.rs index 86fa437159..7ff1bd2e0c 100644 --- a/crates/rust-client/src/sync/block_header.rs +++ b/crates/rust-client/src/sync/block_header.rs @@ -121,14 +121,6 @@ impl Client { /// Tracks each fetched note block in `partial_mmr` and stores its header together with the /// authentication nodes that tracking produced. - /// - /// Tracking is what verifies the node's proof path against the current peaks, so every block is - /// tracked before the first insert: a path that does not verify fails with nothing written. - /// Blocks already tracked are skipped, which covers a block the client picked up in an earlier - /// sync. - /// - /// The caller loads the MMR and caches it afterwards, since the inserts change the tracked - /// block set. pub(crate) async fn insert_note_blocks( &mut self, blocks: BTreeMap, diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index ab8eaaf57d..3310650b83 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -153,12 +153,6 @@ where /// Builds the default sync input and runs [`StateSync::fetch_state`]. The nullifier check is /// not part of this: run [`StateSync::fetch_nullifiers`] on the result before applying it, so /// it can also cover transport-delivered notes another sync path fetched in the same call. - /// - /// The [`StateSync`] is returned with the data because it must stay in scope until the update - /// is applied: its note observers accumulate per-note state during the fetch and drain it in - /// their apply hook, so [`Client::apply_chain_updates`] has to run against the same instances. - /// - /// Takes `&self` so it can run concurrently with the note transport sync's fetch phase. pub async fn fetch_chain_updates(&self) -> Result<(StateSync, ChainSyncData), ClientError> { // Each `NoteObserver` owns its own per-sync state; `with_note_observer` just attaches. let note_screener = self.note_screener(); @@ -175,13 +169,9 @@ where } /// Verifies fetched chain data against the client's partial MMR and writes the resulting - /// update. - /// - /// `state_sync` must be the one that produced `chain_sync_data`: its note observers hold the - /// state they accumulated during the fetch, and their apply hooks run here. + /// update to the store. /// - /// Also caches the partial MMR and prunes irrelevant blocks according to the configured - /// cadence, in that order: pruning reuses the cached MMR. + /// Also caches the partial MMR and prunes irrelevant blocks. /// /// # Errors /// @@ -245,8 +235,8 @@ where /// Runs the full client sync: private notes from the Note Transport Layer and the client's /// on-chain state with the Miden node. /// - /// The two are fetched concurrently, since the transport pages and the node's sync data are - /// independent, and everything that writes runs sequentially afterwards: + /// The NTL and the node are fetched concurrently, and everything that writes runs sequentially + /// afterwards: /// /// 1. Concurrently: the note transport fetch and [`Client::fetch_chain_updates`]. Only node and /// NTL calls happen here, which is all that benefits from overlapping. diff --git a/crates/testing/miden-client-tests/src/tests/transport.rs b/crates/testing/miden-client-tests/src/tests/transport.rs index c53a46ccc3..5b82177f65 100644 --- a/crates/testing/miden-client-tests/src/tests/transport.rs +++ b/crates/testing/miden-client-tests/src/tests/transport.rs @@ -696,16 +696,9 @@ async fn ntl_note_committed_within_the_sync_window_is_committed_by_that_sync() { ); } -/// A note delivered over the NTL whose nullifier is already on chain must not land consumable. +/// A note delivered over the NTL whose nullifier is already on chain must be stored as consumed. /// -/// Probe for 0xMiden/rust-sdk#2422. Commitment discovery and spend discovery have opposite time -/// orientations: the transport import looks *backwards* from the sync height for the commitment -/// (`sync_expected_notes`, plus the sender's block hint), while spend discovery only ever looks -/// *forwards*, `sync_nullifiers(prefixes, checkpoint + 1, tip)`. A note spent below the checkpoint -/// therefore imports as `Committed` and nothing later corrects it. -/// -/// The scenario is a seed restore: the transport re-serves its whole backlog to a cursor-0 client -/// whose checkpoint is already at the tip. +/// Probe for 0xMiden/rust-sdk#2422. #[tokio::test] async fn ntl_note_already_spent_below_the_checkpoint_is_not_left_committed() { let sender_id: AccountId = ACCOUNT_ID_SENDER.try_into().unwrap(); From 4cb2afad8c5a27e93f96923a25488176affdefe6 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sat, 29 Aug 2026 19:43:18 -0300 Subject: [PATCH 19/43] refactor(rust-client): key the awaiting-block notes on their record state --- crates/rust-client/src/note/import.rs | 100 +++++++++++--------------- 1 file changed, 42 insertions(+), 58 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 6b1ce497f5..5a06bb88ab 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -254,20 +254,18 @@ where .map(ResolvedNoteContent::into_attachments) .filter(|attachments| !attachments.is_empty()); - let metadata = *committed_note.metadata(); - let mut changed = note_record - .inclusion_proof_received(committed_note.inclusion_proof().clone(), metadata)?; + // Leaves the record `Unverified`: it now carries the inclusion proof and metadata but + // not the block header that verifies them. `fetch_note_blocks` finishes it. + note_record.inclusion_proof_received( + committed_note.inclusion_proof().clone(), + *committed_note.metadata(), + )?; if let Some(attachments) = attachments { - changed |= note_record.attachments_received(attachments); + note_record.attachments_received(attachments); } - note_updates.committed_notes_awaiting_blocks.push(CommittedNoteAwaitingBlock { - note_record, - block_num: committed_note.block_num(), - committed_tag: metadata.tag(), - changed, - }); + note_updates.notes_to_write.push(note_record); } Ok(note_updates) @@ -287,11 +285,10 @@ where &self, note_updates: &mut TransportNoteUpdates, ) -> Result<(), ClientError> { - let requested_blocks: BTreeSet = note_updates - .committed_notes_awaiting_blocks - .iter() - .map(|note| note.block_num) - .collect(); + // A record left `Unverified` by `fetch_transport_notes_onchain_state` is one the node + // reported as committed: it holds the inclusion proof but not the header verifying it. + let requested_blocks: BTreeSet = + note_updates.notes_to_write.iter().filter_map(awaiting_block_header).collect(); if requested_blocks.is_empty() { return Ok(()); @@ -323,25 +320,26 @@ where ); } - for mut note_awaiting_block in - core::mem::take(&mut note_updates.committed_notes_awaiting_blocks) - { + for note_record in &mut note_updates.notes_to_write { + let Some(block_num) = awaiting_block_header(note_record) else { + continue; + }; let block_header = block_headers - .get(¬e_awaiting_block.block_num) + .get(&block_num) .expect("every committed note's block was fetched above"); - // `block_header_received` transitions the record's state, so it must always run. - note_awaiting_block.changed |= - note_awaiting_block.note_record.block_header_received(block_header)?; - - // A record the block header left unchanged has nothing to write. - if note_awaiting_block.changed { - // Once committed, the note no longer needs its expected-note tag. - note_updates.tags_to_remove.push(NoteTagRecord::with_note_source( - note_awaiting_block.committed_tag, - note_awaiting_block.note_record.details_commitment(), - )); - note_updates.notes_to_write.push(note_awaiting_block.note_record); + // Read before the transition, which moves the record out of the state holding these. + let committed_tag = note_record + .metadata() + .expect("a note awaiting its block header carries metadata") + .tag(); + let details_commitment = note_record.details_commitment(); + + // Once committed, the note no longer needs its expected-note tag. + if note_record.block_header_received(block_header)? { + note_updates + .tags_to_remove + .push(NoteTagRecord::with_note_source(committed_tag, details_commitment)); } } @@ -731,34 +729,13 @@ where // EXPECTED NOTE IMPORT // ================================================================================================ -/// An expected note the node reported as committed, with its inclusion proof and attachments -/// already applied. -/// -/// Until [`Client::fetch_note_blocks`] resolves the block that committed it, the record is -/// missing the block-header transition, so it cannot be written yet. -struct CommittedNoteAwaitingBlock { - /// The record. Carries every transition but the block header until the block is resolved. - note_record: InputNoteRecord, - /// Block that committed the note. - block_num: BlockNumber, - /// Note-source tag to drop, since a committed note no longer needs to be watched for. - committed_tag: NoteTag, - /// Whether the inclusion-proof and attachment transitions already changed the record. - changed: bool, -} - -/// Notes fetched from the Note Transport Layer, split by whether the node has -/// committed them, along with note tags to remove, and the blocks that must be -/// stored before their corresponding committed notes. +/// Notes fetched from the Note Transport Layer, along with note tags to remove, and the blocks +/// that must be stored before their corresponding committed notes. #[derive(Default)] pub(crate) struct TransportNoteUpdates { - /// Records ready to write: the notes the node has not committed, plus the committed ones - /// once [`Client::fetch_note_blocks`] has resolved their blocks. + /// The records to write. A note the node has not committed stays `Expected`; one it has + /// committed is `Unverified` until [`Client::fetch_note_blocks`] supplies its block header. notes_to_write: Vec, - /// Notes the node has committed, each waiting on the block that committed it. - /// [`Client::fetch_note_blocks`] drains these into `notes_to_write`, dropping the ones the - /// block header leaves unchanged, so this is empty by the time the batch is written. - committed_notes_awaiting_blocks: Vec, /// Blocks that must be tracked and stored before the committed notes that need them, keyed by /// block number so a block committing several notes is stored once. Filled by /// [`Client::fetch_note_blocks`]; blocks the client already tracks are absent. @@ -783,8 +760,6 @@ impl TransportNoteUpdates { /// resolve to the version fetched last. pub(crate) fn merge(&mut self, other: Self) { self.notes_to_write.extend(other.notes_to_write); - self.committed_notes_awaiting_blocks - .extend(other.committed_notes_awaiting_blocks); self.blocks_to_insert.extend(other.blocks_to_insert); self.tags_to_remove.extend(other.tags_to_remove); } @@ -801,6 +776,15 @@ impl TransportNoteUpdates { // HELPERS // ================================================================================================ +/// The block that committed a note whose record is still awaiting its header, or `None` for any +/// other record. +fn awaiting_block_header(note_record: &InputNoteRecord) -> Option { + match note_record.state() { + InputNoteState::Unverified(state) => Some(state.inclusion_proof.location().block_num()), + _ => None, + } +} + /// Returns an error if the already-stored note is currently being processed by a local /// transaction, since an in-flight note can't be overwritten by an import. fn ensure_not_processing(previous_note: Option<&InputNoteRecord>) -> Result<(), ClientError> { From f442817f1d8a36397aa5efbe09df0f05af7f9ee9 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sat, 29 Aug 2026 19:47:52 -0300 Subject: [PATCH 20/43] docs(rust-client): correct the note import docs for the single note list --- crates/rust-client/src/note/import.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 5a06bb88ab..4844c9c3dd 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -349,8 +349,9 @@ where /// Fetches the nullifier commit heights of the notes in `note_updates`, marking those already /// spent as consumed. /// - /// Must run after [`Client::fetch_note_blocks`]: a note only has a nullifier once it is - /// committed, so before that there is nothing to ask about. + /// Must run after [`Client::fetch_note_blocks`]: only a note the node reported as committed + /// has the metadata a nullifier is derived from, so before that there is nothing to ask + /// about. pub(crate) async fn fetch_note_nullifiers( &self, note_updates: &mut TransportNoteUpdates, @@ -395,8 +396,8 @@ where /// of the written records. /// /// The block headers go in first, so a record is never persisted as committed before the - /// header proving its inclusion. [`Client::fetch_note_blocks`] must have run beforehand: a - /// note still awaiting its block is not written. + /// header proving its inclusion. [`Client::fetch_note_blocks`] must have run beforehand, or a + /// note the node committed is written as unverified and stays that way until a later sync. pub(crate) async fn apply_note_transport_updates( &mut self, note_updates: TransportNoteUpdates, @@ -764,10 +765,7 @@ impl TransportNoteUpdates { self.tags_to_remove.extend(other.tags_to_remove); } - /// The records this batch is about to write. - /// - /// Only complete once [`Client::fetch_note_blocks`] has run: before that, the - /// committed notes are still awaiting their blocks and are not included. + /// The records this batch is about to write, whatever state each is in. pub(crate) fn input_note_records(&self) -> impl Iterator { self.notes_to_write.iter() } From 603e7c7699a3f33cf49a77947ef4b52393416408 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sat, 29 Aug 2026 20:24:49 -0300 Subject: [PATCH 21/43] refactor(rust-client): import notes by details through the transport fetch/apply path --- crates/rust-client/src/note/import.rs | 430 +++++++++++--------------- 1 file changed, 183 insertions(+), 247 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 4844c9c3dd..8112c505ce 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -126,10 +126,9 @@ where match note_file { NoteFile::ExpectedNote { details, sync_hint } => { requests_by_details.push(( - previous_note, details, sync_hint.after_block_num(), - Some(sync_hint.tag()), + sync_hint.tag(), )); }, NoteFile::Committed { note, proof } => { @@ -141,6 +140,7 @@ where } } + let mut imported_commitments = vec![]; let mut imported_notes = vec![]; if !requests_by_id.is_empty() { let notes_by_id = self.import_note_records_by_id(requests_by_id).await?; @@ -148,8 +148,8 @@ where } if !requests_by_details.is_empty() { - let notes_by_details = self.import_note_records_by_details(requests_by_details).await?; - imported_notes.extend(notes_by_details); + let commitments = self.import_note_records_by_details(requests_by_details).await?; + imported_commitments.extend(commitments); } if !requests_by_proof.is_empty() { @@ -157,7 +157,7 @@ where imported_notes.extend(notes_by_proof); } - let mut imported_commitments = Vec::with_capacity(imported_notes.len()); + imported_commitments.reserve(imported_notes.len()); for note in imported_notes { let details_commitment = note.details_commitment(); if let InputNoteState::Expected(ExpectedNoteState { tag: Some(tag), .. }) = note.state() @@ -173,6 +173,184 @@ where Ok(imported_commitments) } + // HELPERS + // ================================================================================================ + + /// Builds note records from the note IDs. If a note with the same ID was already stored it + /// is passed via `previous_note` so it can be updated. The note information is fetched from + /// the node and stored in the client's store. + /// + /// Only records that changed as a result of the import are returned. + /// + /// # Errors: + /// - If a note doesn't exist on the node. + /// - If a note exists but is private. + async fn import_note_records_by_id( + &mut self, + notes: BTreeMap>, + ) -> Result, ClientError> { + let note_ids = notes.keys().copied().collect::>(); + + let fetched_notes = + self.rpc_api.get_notes_by_id(¬e_ids).await.map_err(|err| match err { + RpcError::NoteNotFound(note_id) => ClientError::NoteNotFoundOnChain(note_id), + err => ClientError::RpcError(err), + })?; + + if fetched_notes.is_empty() { + return Err(ClientError::NoteImportError("No notes fetched from node".to_string())); + } + + let mut note_records = Vec::new(); + let mut notes_to_request = vec![]; + for fetched_note in fetched_notes { + let note_id = fetched_note.id(); + let inclusion_proof = fetched_note.inclusion_proof().clone(); + + let previous_note = + notes.get(¬e_id).cloned().ok_or(ClientError::NoteImportError(format!( + "Failed to retrieve note with id {note_id} from node" + )))?; + if let Some(mut previous_note) = previous_note { + if previous_note + .inclusion_proof_received(inclusion_proof, *fetched_note.metadata())? + { + self.store.remove_note_tag((&previous_note).try_into()?).await?; + + note_records.push(previous_note); + } + } else { + let fetched_note = match fetched_note { + FetchedNote::Public(note, _) => note, + FetchedNote::Private(..) => { + return Err(ClientError::NoteImportError( + "Incomplete imported note is private".to_string(), + )); + }, + }; + + let note_request = (previous_note, fetched_note, inclusion_proof); + notes_to_request.push(note_request); + } + } + + if !notes_to_request.is_empty() { + let note_records_by_proof = self.import_note_records_by_proof(notes_to_request).await?; + note_records.extend(note_records_by_proof); + } + Ok(note_records) + } + + /// Builds a note record list from notes and inclusion proofs. If a note with the same ID was + /// already stored it is passed via `previous_note` so it can be updated. The note's + /// nullifier is used to determine if the note has been consumed in the node and gives it + /// the correct state. + /// + /// If the note isn't consumed and it was committed in the past relative to the client, then + /// the MMR for the relevant block is fetched from the node and stored. + /// + /// Only records that changed as a result of the import are returned. + pub(crate) async fn import_note_records_by_proof( + &mut self, + requested_notes: Vec<(Option, Note, NoteInclusionProof)>, + ) -> Result, ClientError> { + // TODO: iterating twice over requested notes + let mut note_records = vec![]; + + let mut nullifier_requests = BTreeSet::new(); + let mut lowest_block_height: BlockNumber = u32::MAX.into(); + for (previous_note, note, inclusion_proof) in &requested_notes { + let nullifier = match previous_note { + Some(previous_note) => previous_note.nullifier(), + None => Some(note.nullifier()), + }; + if let Some(nullifier) = nullifier { + nullifier_requests.insert(nullifier); + } + if inclusion_proof.location().block_num() < lowest_block_height { + lowest_block_height = inclusion_proof.location().block_num(); + } + } + + let nullifier_commit_heights = self + .rpc_api + .get_nullifier_commit_heights(nullifier_requests, lowest_block_height) + .await?; + let mut partial_mmr = self.get_current_partial_mmr().await?; + + for (previous_note, note, inclusion_proof) in requested_notes { + let metadata = *note.metadata(); + let attachments = note.attachments().clone(); + let mut note_record = previous_note.unwrap_or(InputNoteRecord::new( + note.into(), + attachments, + self.store.get_current_timestamp(), + ExpectedNoteState { + metadata: Some(metadata), + after_block_num: inclusion_proof.location().block_num(), + tag: Some(metadata.tag()), + } + .into(), + )); + + if let Some(nullifier) = note_record.nullifier() + && let Some(Some(block_height)) = nullifier_commit_heights.get(&nullifier) + { + if note_record.consumed_externally(nullifier, *block_height, None)? { + note_records.push(note_record); + } + } else { + let block_height = inclusion_proof.location().block_num(); + let current_block_num = self.get_sync_height().await?; + + let tag = metadata.tag(); + let mut note_changed = + note_record.inclusion_proof_received(inclusion_proof, metadata)?; + + if block_height <= current_block_num { + // A note committed in the past needs its block header fetched and + // authenticated to verify the inclusion proof. + let block_header = self + .get_and_store_authenticated_block(block_height, &mut partial_mmr) + .await?; + note_changed |= note_record.block_header_received(&block_header)?; + } else { + // If the note is in the future we import it as unverified. We add the note tag + // so that the note is verified naturally in the next sync. + self.store + .add_note_tag(NoteTagRecord::with_note_source( + tag, + note_record.details_commitment(), + )) + .await?; + } + + if note_changed { + note_records.push(note_record); + } + } + } + self.cache_partial_mmr(partial_mmr).await?; + + Ok(note_records) + } + + /// Imports notes from their details, storing the resulting records. + /// + /// Notes the node has not reported as committed get (or keep) their expected record; the rest + /// are stored as committed, together with the blocks proving their inclusion. + /// + /// The `requested_notes` parameter carries, for each note, its details, the block from which + /// its commitment should be looked for, and the tag to track it under. + async fn import_note_records_by_details( + &mut self, + requested_notes: Vec<(NoteDetails, BlockNumber, NoteTag)>, + ) -> Result, ClientError> { + let mut note_updates = self.fetch_transport_notes_onchain_state(&requested_notes).await?; + self.fetch_note_blocks(&mut note_updates).await?; + self.apply_note_transport_updates(note_updates).await + } + // TRANSPORT-DELIVERED NOTE IMPORT // -------------------------------------------------------------------------------------------- @@ -429,248 +607,6 @@ where Ok(written) } - // HELPERS - // ================================================================================================ - - /// Builds note records from the note IDs. If a note with the same ID was already stored it - /// is passed via `previous_note` so it can be updated. The note information is fetched from - /// the node and stored in the client's store. - /// - /// Only records that changed as a result of the import are returned. - /// - /// # Errors: - /// - If a note doesn't exist on the node. - /// - If a note exists but is private. - async fn import_note_records_by_id( - &mut self, - notes: BTreeMap>, - ) -> Result, ClientError> { - let note_ids = notes.keys().copied().collect::>(); - - let fetched_notes = - self.rpc_api.get_notes_by_id(¬e_ids).await.map_err(|err| match err { - RpcError::NoteNotFound(note_id) => ClientError::NoteNotFoundOnChain(note_id), - err => ClientError::RpcError(err), - })?; - - if fetched_notes.is_empty() { - return Err(ClientError::NoteImportError("No notes fetched from node".to_string())); - } - - let mut note_records = Vec::new(); - let mut notes_to_request = vec![]; - for fetched_note in fetched_notes { - let note_id = fetched_note.id(); - let inclusion_proof = fetched_note.inclusion_proof().clone(); - - let previous_note = - notes.get(¬e_id).cloned().ok_or(ClientError::NoteImportError(format!( - "Failed to retrieve note with id {note_id} from node" - )))?; - if let Some(mut previous_note) = previous_note { - if previous_note - .inclusion_proof_received(inclusion_proof, *fetched_note.metadata())? - { - self.store.remove_note_tag((&previous_note).try_into()?).await?; - - note_records.push(previous_note); - } - } else { - let fetched_note = match fetched_note { - FetchedNote::Public(note, _) => note, - FetchedNote::Private(..) => { - return Err(ClientError::NoteImportError( - "Incomplete imported note is private".to_string(), - )); - }, - }; - - let note_request = (previous_note, fetched_note, inclusion_proof); - notes_to_request.push(note_request); - } - } - - if !notes_to_request.is_empty() { - let note_records_by_proof = self.import_note_records_by_proof(notes_to_request).await?; - note_records.extend(note_records_by_proof); - } - Ok(note_records) - } - - /// Builds a note record list from notes and inclusion proofs. If a note with the same ID was - /// already stored it is passed via `previous_note` so it can be updated. The note's - /// nullifier is used to determine if the note has been consumed in the node and gives it - /// the correct state. - /// - /// If the note isn't consumed and it was committed in the past relative to the client, then - /// the MMR for the relevant block is fetched from the node and stored. - /// - /// Only records that changed as a result of the import are returned. - pub(crate) async fn import_note_records_by_proof( - &mut self, - requested_notes: Vec<(Option, Note, NoteInclusionProof)>, - ) -> Result, ClientError> { - // TODO: iterating twice over requested notes - let mut note_records = vec![]; - - let mut nullifier_requests = BTreeSet::new(); - let mut lowest_block_height: BlockNumber = u32::MAX.into(); - for (previous_note, note, inclusion_proof) in &requested_notes { - let nullifier = match previous_note { - Some(previous_note) => previous_note.nullifier(), - None => Some(note.nullifier()), - }; - if let Some(nullifier) = nullifier { - nullifier_requests.insert(nullifier); - } - if inclusion_proof.location().block_num() < lowest_block_height { - lowest_block_height = inclusion_proof.location().block_num(); - } - } - - let nullifier_commit_heights = self - .rpc_api - .get_nullifier_commit_heights(nullifier_requests, lowest_block_height) - .await?; - let mut partial_mmr = self.get_current_partial_mmr().await?; - - for (previous_note, note, inclusion_proof) in requested_notes { - let metadata = *note.metadata(); - let attachments = note.attachments().clone(); - let mut note_record = previous_note.unwrap_or(InputNoteRecord::new( - note.into(), - attachments, - self.store.get_current_timestamp(), - ExpectedNoteState { - metadata: Some(metadata), - after_block_num: inclusion_proof.location().block_num(), - tag: Some(metadata.tag()), - } - .into(), - )); - - if let Some(nullifier) = note_record.nullifier() - && let Some(Some(block_height)) = nullifier_commit_heights.get(&nullifier) - { - if note_record.consumed_externally(nullifier, *block_height, None)? { - note_records.push(note_record); - } - } else { - let block_height = inclusion_proof.location().block_num(); - let current_block_num = self.get_sync_height().await?; - - let tag = metadata.tag(); - let mut note_changed = - note_record.inclusion_proof_received(inclusion_proof, metadata)?; - - if block_height <= current_block_num { - // A note committed in the past needs its block header fetched and - // authenticated to verify the inclusion proof. - let block_header = self - .get_and_store_authenticated_block(block_height, &mut partial_mmr) - .await?; - note_changed |= note_record.block_header_received(&block_header)?; - } else { - // If the note is in the future we import it as unverified. We add the note tag - // so that the note is verified naturally in the next sync. - self.store - .add_note_tag(NoteTagRecord::with_note_source( - tag, - note_record.details_commitment(), - )) - .await?; - } - - if note_changed { - note_records.push(note_record); - } - } - } - self.cache_partial_mmr(partial_mmr).await?; - - Ok(note_records) - } - - /// Builds a note record list from note details. If a note with the same ID was already stored - /// it is passed via `previous_note` so it can be updated. - /// - /// Only records that need to be stored are returned: notes the node has not reported as - /// committed keep (or get) their expected record, while committed notes are returned only if - /// the new information changed them. - async fn import_note_records_by_details( - &mut self, - requested_notes: Vec<(Option, NoteDetails, BlockNumber, Option)>, - ) -> Result, ClientError> { - let mut lowest_request_block: BlockNumber = u32::MAX.into(); - let mut note_requests = vec![]; - for (_, details, after_block_num, tag) in &requested_notes { - if let Some(tag) = tag { - note_requests.push((details.commitment(), *tag)); - lowest_request_block = lowest_request_block.min(*after_block_num); - } - } - let mut committed_notes_data = - self.sync_expected_notes(lowest_request_block, note_requests).await?; - - let mut note_records = vec![]; - let mut partial_mmr = self.get_current_partial_mmr().await?; - - for (previous_note, details, after_block_num, tag) in requested_notes { - let mut note_record = previous_note.unwrap_or_else(|| { - InputNoteRecord::new( - details, - NoteAttachments::empty(), - self.store.get_current_timestamp(), - ExpectedNoteState { metadata: None, after_block_num, tag }.into(), - ) - }); - - // Notes the node has not reported as committed keep their expected record untouched. - let Some(SyncedNote { committed: committed_note, content }) = - committed_notes_data.remove(¬e_record.details_commitment()) - else { - note_records.push(note_record); - continue; - }; - - let attachments = content - .map(ResolvedNoteContent::into_attachments) - .filter(|attachments| !attachments.is_empty()); - - let block_header = self - .get_and_store_authenticated_block(committed_note.block_num(), &mut partial_mmr) - .await?; - - let metadata = *committed_note.metadata(); - let mut note_changed = note_record - .inclusion_proof_received(committed_note.inclusion_proof().clone(), metadata)?; - - if let Some(attachments) = attachments { - note_changed |= note_record.attachments_received(attachments); - } - - // `block_header_received` transitions the record's state, so it must always run. - note_changed |= note_record.block_header_received(&block_header)?; - - // Once committed, the note no longer needs its expected-note tag. - if note_changed { - self.store - .remove_note_tag(NoteTagRecord::with_note_source( - metadata.tag(), - note_record.details_commitment(), - )) - .await?; - } - - if note_changed { - note_records.push(note_record); - } - } - self.cache_partial_mmr(partial_mmr).await?; - - Ok(note_records) - } - /// Checks whether the expected notes (identified by their details commitments and tags) have /// been committed on chain between `request_block_num` and the current block, returning the /// matching synced notes keyed by details commitment. From daf6dc871218ebee87ec7c405b80b3d2d4503dac Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sat, 29 Aug 2026 20:35:53 -0300 Subject: [PATCH 22/43] refactor(rust-client): match the transport note fetch interface to the by-details import --- crates/rust-client/src/note/import.rs | 91 +++++++------------- crates/rust-client/src/note/mod.rs | 2 +- crates/rust-client/src/note_transport/mod.rs | 28 ++++-- 3 files changed, 56 insertions(+), 65 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 8112c505ce..42f90f4a83 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -126,6 +126,7 @@ where match note_file { NoteFile::ExpectedNote { details, sync_hint } => { requests_by_details.push(( + previous_note, details, sync_hint.after_block_num(), sync_hint.tag(), @@ -344,9 +345,9 @@ where /// its commitment should be looked for, and the tag to track it under. async fn import_note_records_by_details( &mut self, - requested_notes: Vec<(NoteDetails, BlockNumber, NoteTag)>, + requested_notes: Vec<(Option, NoteDetails, BlockNumber, NoteTag)>, ) -> Result, ClientError> { - let mut note_updates = self.fetch_transport_notes_onchain_state(&requested_notes).await?; + let mut note_updates = self.fetch_transport_notes_onchain_state(requested_notes).await?; self.fetch_note_blocks(&mut note_updates).await?; self.apply_note_transport_updates(note_updates).await } @@ -354,75 +355,45 @@ where // TRANSPORT-DELIVERED NOTE IMPORT // -------------------------------------------------------------------------------------------- - /// Fetches the state of transport-delivered notes from the RPC, and returns a - /// [`TransportNoteUpdates`] containing the expected notes that are ready to write to the - /// store, along with the committed notes that wait for [`Client::fetch_note_blocks`] to - /// resolve their blocks. + /// Fetches the on-chain state of transport-delivered notes, returning the records to write + /// and the blocks that still have to be resolved by [`Client::fetch_note_blocks`]. /// - /// The `requests` parameter contains, for each transport-delivered note, the note details, - /// the block from which its commitment should be looked for, and the tag to track it under. - /// - /// # Errors - /// - /// - If a note being imported is currently being processed by a local transaction. + /// A note with a stored version is passed via `previous_note` so it can be updated. Notes the + /// node has not reported as committed keep (or get) their expected record; the rest are left + /// `Unverified`, carrying the inclusion proof but not the header that verifies it. pub(crate) async fn fetch_transport_notes_onchain_state( &self, - requests: &[(NoteDetails, BlockNumber, NoteTag)], + requested_notes: Vec<(Option, NoteDetails, BlockNumber, NoteTag)>, ) -> Result { - let mut note_updates = TransportNoteUpdates::default(); - if requests.is_empty() { - return Ok(note_updates); - } - - // Deduplicate by details commitment, keeping the last request for each note. - let mut requests_by_commitment = BTreeMap::new(); - for (details, after_block_num, tag) in requests { - requests_by_commitment - .insert(details.commitment(), (details.clone(), *after_block_num, *tag)); - } - - let previous_by_commitment: BTreeMap = self - .get_input_notes(NoteFilter::DetailsCommitments( - requests_by_commitment.keys().copied().collect(), - )) - .await? - .into_iter() - .map(|note| (note.details_commitment(), note)) - .collect(); - - // Validate before building anything, so a single in-flight note aborts the whole import. - for previous_note in previous_by_commitment.values() { - ensure_not_processing(Some(previous_note))?; - } - let mut lowest_request_block: BlockNumber = u32::MAX.into(); - let mut note_requests = Vec::with_capacity(requests_by_commitment.len()); - for (commitment, (_, after_block_num, tag)) in &requests_by_commitment { - note_requests.push((*commitment, *tag)); + let mut note_requests = vec![]; + for (_, details, after_block_num, tag) in &requested_notes { + note_requests.push((details.commitment(), *tag)); lowest_request_block = lowest_request_block.min(*after_block_num); } let mut committed_notes_data = self.sync_expected_notes(lowest_request_block, note_requests).await?; - for (commitment, (details, after_block_num, tag)) in requests_by_commitment { - let mut note_record = - previous_by_commitment.get(&commitment).cloned().unwrap_or_else(|| { - InputNoteRecord::new( - details, - NoteAttachments::empty(), - self.store.get_current_timestamp(), - ExpectedNoteState { - metadata: None, - after_block_num, - tag: Some(tag), - } - .into(), - ) - }); + let mut note_updates = TransportNoteUpdates::default(); + + for (previous_note, details, after_block_num, tag) in requested_notes { + let mut note_record = previous_note.unwrap_or_else(|| { + InputNoteRecord::new( + details, + NoteAttachments::empty(), + self.store.get_current_timestamp(), + ExpectedNoteState { + metadata: None, + after_block_num, + tag: Some(tag), + } + .into(), + ) + }); // Notes the node has not reported as committed keep their expected record untouched. let Some(SyncedNote { committed: committed_note, content }) = - committed_notes_data.remove(&commitment) + committed_notes_data.remove(¬e_record.details_commitment()) else { note_updates.notes_to_write.push(note_record); continue; @@ -721,7 +692,9 @@ fn awaiting_block_header(note_record: &InputNoteRecord) -> Option { /// Returns an error if the already-stored note is currently being processed by a local /// transaction, since an in-flight note can't be overwritten by an import. -fn ensure_not_processing(previous_note: Option<&InputNoteRecord>) -> Result<(), ClientError> { +pub(crate) fn ensure_not_processing( + previous_note: Option<&InputNoteRecord>, +) -> Result<(), ClientError> { if let Some(note) = previous_note && note.is_processing() { diff --git a/crates/rust-client/src/note/mod.rs b/crates/rust-client/src/note/mod.rs index 6552f0c256..be92bff56b 100644 --- a/crates/rust-client/src/note/mod.rs +++ b/crates/rust-client/src/note/mod.rs @@ -69,7 +69,7 @@ use crate::store::{InputNoteRecord, NoteFilter, OutputNoteRecord}; use crate::{Client, ClientError, IdPrefixFetchError}; mod import; -pub(crate) use import::{NoteBlockToInsert, TransportNoteUpdates}; +pub(crate) use import::{NoteBlockToInsert, TransportNoteUpdates, ensure_not_processing}; mod note_reader; mod note_screener; mod note_update_tracker; diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 8e0a137f14..5023d0c3f5 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -24,8 +24,8 @@ use miden_tx::utils::serde::{ }; pub use self::errors::NoteTransportError; -use crate::note::TransportNoteUpdates; -use crate::store::InputNoteRecord; +use crate::note::{TransportNoteUpdates, ensure_not_processing}; +use crate::store::{InputNoteRecord, NoteFilter}; use crate::sync::NoteTagSource; use crate::{Client, ClientError}; @@ -484,15 +484,33 @@ where let fallback_after_block_num = BlockNumber::from(sync_height.as_u32().saturating_sub(NOTE_LOOKBACK_BLOCKS)); - let mut requests = Vec::with_capacity(notes.len()); + // Deduplicate by details commitment, so a note delivered twice is requested once. + let mut requests_by_commitment = BTreeMap::new(); for (note, block_hint) in notes { let tag = note.metadata().tag(); // Prefer the sender-provided hint, falling back to the lookback window when absent. let after_block_num = block_hint.unwrap_or(fallback_after_block_num); - requests.push((NoteDetails::from(note), after_block_num, tag)); + let details = NoteDetails::from(note); + requests_by_commitment.insert(details.commitment(), (details, after_block_num, tag)); } - let note_updates = self.fetch_transport_notes_onchain_state(&requests).await?; + let mut previous_by_commitment: BTreeMap = self + .get_input_notes(NoteFilter::DetailsCommitments( + requests_by_commitment.keys().copied().collect(), + )) + .await? + .into_iter() + .map(|note| (note.details_commitment(), note)) + .collect(); + + let mut requests = Vec::with_capacity(requests_by_commitment.len()); + for (commitment, (details, after_block_num, tag)) in requests_by_commitment { + let previous_note = previous_by_commitment.remove(&commitment); + ensure_not_processing(previous_note.as_ref())?; + requests.push((previous_note, details, after_block_num, tag)); + } + + let note_updates = self.fetch_transport_notes_onchain_state(requests).await?; Ok((note_updates, rcursor)) } From 6e4fcb5e891f001eb592068566e71cfd026c06f1 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sat, 29 Aug 2026 20:57:13 -0300 Subject: [PATCH 23/43] refactor(rust-client): return the by-details import records for import_notes to store --- crates/rust-client/src/note/import.rs | 32 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 42f90f4a83..6b2365445e 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -141,7 +141,6 @@ where } } - let mut imported_commitments = vec![]; let mut imported_notes = vec![]; if !requests_by_id.is_empty() { let notes_by_id = self.import_note_records_by_id(requests_by_id).await?; @@ -149,8 +148,8 @@ where } if !requests_by_details.is_empty() { - let commitments = self.import_note_records_by_details(requests_by_details).await?; - imported_commitments.extend(commitments); + let notes_by_details = self.import_note_records_by_details(requests_by_details).await?; + imported_notes.extend(notes_by_details); } if !requests_by_proof.is_empty() { @@ -158,7 +157,7 @@ where imported_notes.extend(notes_by_proof); } - imported_commitments.reserve(imported_notes.len()); + let mut imported_commitments = Vec::with_capacity(imported_notes.len()); for note in imported_notes { let details_commitment = note.details_commitment(); if let InputNoteState::Expected(ExpectedNoteState { tag: Some(tag), .. }) = note.state() @@ -336,20 +335,29 @@ where Ok(note_records) } - /// Imports notes from their details, storing the resulting records. + /// Builds a note record list from note details. If a note with the same ID was already stored + /// it is passed via `previous_note` so it can be updated. /// - /// Notes the node has not reported as committed get (or keep) their expected record; the rest - /// are stored as committed, together with the blocks proving their inclusion. - /// - /// The `requested_notes` parameter carries, for each note, its details, the block from which - /// its commitment should be looked for, and the tag to track it under. + /// Only records that need to be stored are returned: notes the node has not reported as + /// committed keep (or get) their expected record, while committed notes are returned only if + /// the new information changed them. async fn import_note_records_by_details( &mut self, requested_notes: Vec<(Option, NoteDetails, BlockNumber, NoteTag)>, - ) -> Result, ClientError> { + ) -> Result, ClientError> { let mut note_updates = self.fetch_transport_notes_onchain_state(requested_notes).await?; self.fetch_note_blocks(&mut note_updates).await?; - self.apply_note_transport_updates(note_updates).await + + let mut partial_mmr = self.get_current_partial_mmr().await?; + self.insert_note_blocks(note_updates.blocks_to_insert, &mut partial_mmr).await?; + // Cache MMR so pruning can reuse in-memory MMR. + self.cache_partial_mmr(partial_mmr).await?; + + for tag in note_updates.tags_to_remove { + self.store.remove_note_tag(tag).await?; + } + + Ok(note_updates.notes_to_write) } // TRANSPORT-DELIVERED NOTE IMPORT From e84330913b57ef23bf60382fc765974b40c0088b Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sun, 30 Aug 2026 10:52:03 -0300 Subject: [PATCH 24/43] refactor(rust-client): simplify the sync_state phase boundaries --- crates/rust-client/src/note/import.rs | 21 +++------ crates/rust-client/src/note_transport/mod.rs | 23 ++++----- crates/rust-client/src/sync/mod.rs | 49 +++++++++++--------- crates/rust-client/src/sync/state_sync.rs | 15 +++--- 4 files changed, 50 insertions(+), 58 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 6b2365445e..d32500bc8e 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -549,8 +549,7 @@ where Ok(()) } - /// Applies the changes from `note_updates` to the store, returning the details commitments - /// of the written records. + /// Applies the changes from `note_updates` to the store, returning the written records. /// /// The block headers go in first, so a record is never persisted as committed before the /// header proving its inclusion. [`Client::fetch_note_blocks`] must have run beforehand, or a @@ -558,7 +557,7 @@ where pub(crate) async fn apply_note_transport_updates( &mut self, note_updates: TransportNoteUpdates, - ) -> Result, ClientError> { + ) -> Result, ClientError> { let mut partial_mmr = self.get_current_partial_mmr().await?; self.insert_note_blocks(note_updates.blocks_to_insert, &mut partial_mmr).await?; // Cache MMR so pruning can reuse in-memory MMR. @@ -568,22 +567,19 @@ where self.store.remove_note_tag(tag).await?; } - let mut written = Vec::with_capacity(note_updates.notes_to_write.len()); - for note in note_updates.notes_to_write { - let details_commitment = note.details_commitment(); + for note in ¬e_updates.notes_to_write { // A record still expected needs its tag tracked so a later sync finds it. A committed // one is no longer in that state, so it is skipped here. if let InputNoteState::Expected(ExpectedNoteState { tag: Some(tag), .. }) = note.state() { self.store - .add_note_tag(NoteTagRecord::with_note_source(*tag, details_commitment)) + .add_note_tag(NoteTagRecord::with_note_source(*tag, note.details_commitment())) .await?; } - self.store.upsert_input_notes(&[note]).await?; - written.push(details_commitment); } + self.store.upsert_input_notes(¬e_updates.notes_to_write).await?; - Ok(written) + Ok(note_updates.notes_to_write) } /// Checks whether the expected notes (identified by their details commitments and tags) have @@ -679,11 +675,6 @@ impl TransportNoteUpdates { self.blocks_to_insert.extend(other.blocks_to_insert); self.tags_to_remove.extend(other.tags_to_remove); } - - /// The records this batch is about to write, whatever state each is in. - pub(crate) fn input_note_records(&self) -> impl Iterator { - self.notes_to_write.iter() - } } // HELPERS diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 5023d0c3f5..be28948466 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -583,7 +583,10 @@ where } /// Writes everything [`Client::fetch_note_transport_sync_data`] fetched, returning the ids of - /// the imported notes. + /// the imported notes and the records it wrote. + /// + /// The records are returned so the chain sync can track them: they are in the store now, but + /// not in the snapshot its tracker was built from. /// /// The notes are written before the covered-tag set and the cursor, so a crash between them /// re-fetches instead of skipping notes that were never written. @@ -593,7 +596,7 @@ where pub(crate) async fn apply_note_transport_sync_data( &mut self, note_transport_data: NoteTransportSyncData, - ) -> Result, ClientError> { + ) -> Result<(Vec, Vec), ClientError> { let NoteTransportSyncData { covered_tags, note_updates, @@ -603,8 +606,8 @@ where let written = self.apply_note_transport_updates(note_updates).await?; let mut imported_ids: Vec = written - .into_iter() - .filter_map(|commitment| id_by_commitment.get(&commitment).copied()) + .iter() + .filter_map(|note| id_by_commitment.get(¬e.details_commitment()).copied()) .collect(); if let Some(covered_tags) = covered_tags { @@ -618,7 +621,7 @@ where imported_ids.sort_unstable(); imported_ids.dedup(); - Ok(imported_ids) + Ok((imported_ids, written)) } } @@ -643,16 +646,6 @@ pub(crate) struct NoteTransportSyncData { cursor: Option, } -impl NoteTransportSyncData { - /// The records this sync is about to write. - /// - /// Used to extend the chain sync's nullifier check to the transport-delivered notes, - /// which are not in the store yet. - pub(crate) fn input_note_records(&self) -> impl Iterator { - self.note_updates.input_note_records() - } -} - /// Note transport cursor /// /// Pagination integer used to reduce the number of fetched notes from the note transport network, diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 3310650b83..ab4c73c740 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -72,7 +72,7 @@ use miden_tx::utils::serde::{Deserializable, DeserializationError, Serializable} use tracing::{debug, info}; use crate::pswap::PswapChainObserver; -use crate::store::{InputNoteRecord, NoteFilter, TransactionFilter}; +use crate::store::{NoteFilter, TransactionFilter}; use crate::{Client, ClientError}; mod block_header; @@ -139,9 +139,10 @@ where self.ensure_genesis_in_place().await?; self.ensure_rpc_limits_in_place().await?; - let (state_sync, mut chain_sync_data) = self.fetch_chain_updates().await?; + let state_sync = self.state_sync(); + let mut chain_sync_data = self.fetch_chain_updates(&state_sync).await?; // No other sync path ran, so there are no transport-delivered notes to take on. - state_sync.process_fetched_state(&mut chain_sync_data, Vec::new()).await?; + state_sync.screen_fetched_notes(&mut chain_sync_data, Vec::new()).await?; state_sync.fetch_nullifiers(&mut chain_sync_data).await?; self.apply_chain_updates(&state_sync, chain_sync_data).await @@ -153,19 +154,23 @@ where /// Builds the default sync input and runs [`StateSync::fetch_state`]. The nullifier check is /// not part of this: run [`StateSync::fetch_nullifiers`] on the result before applying it, so /// it can also cover transport-delivered notes another sync path fetched in the same call. - pub async fn fetch_chain_updates(&self) -> Result<(StateSync, ChainSyncData), ClientError> { - // Each `NoteObserver` owns its own per-sync state; `with_note_observer` just attaches. - let note_screener = self.note_screener(); - let state_sync = - StateSync::new(self.rpc_api.clone(), Arc::new(note_screener), self.tx_discard_delta) - .with_note_observer(Arc::new(PswapChainObserver::new(self.store.clone()))); - + pub async fn fetch_chain_updates( + &self, + state_sync: &StateSync, + ) -> Result { let input = self.build_sync_input().await?; let block_from = block_num_from_forest(&self.get_current_partial_mmr().await?)?; - let chain_sync_data = state_sync.fetch_state(block_from, input).await?; + state_sync.fetch_state(block_from, input).await + } - Ok((state_sync, chain_sync_data)) + /// Builds the [`StateSync`] driving one chain sync. + /// + /// Each `NoteObserver` owns its own per-sync state, so this must be called once per sync + /// rather than shared; `with_note_observer` just attaches it. + fn state_sync(&self) -> StateSync { + StateSync::new(self.rpc_api.clone(), Arc::new(self.note_screener()), self.tx_discard_delta) + .with_note_observer(Arc::new(PswapChainObserver::new(self.store.clone()))) } /// Verifies fetched chain data against the client's partial MMR and writes the resulting @@ -229,7 +234,8 @@ where let note_transport_data = self.fetch_note_transport_sync_data().await?; - self.apply_note_transport_sync_data(note_transport_data).await + let (imported_ids, _) = self.apply_note_transport_sync_data(note_transport_data).await?; + Ok(imported_ids) } /// Runs the full client sync: private notes from the Note Transport Layer and the client's @@ -241,7 +247,7 @@ where /// 1. Concurrently: the note transport fetch and [`Client::fetch_chain_updates`]. Only node and /// NTL calls happen here, which is all that benefits from overlapping. /// 2. The transport writes. - /// 3. [`StateSync::process_fetched_state`], which screens the node's notes against the store — + /// 3. [`StateSync::screen_fetched_notes`], which screens the node's notes against the store — /// hence after step 2, so a transport-delivered note is recognised rather than discarded — /// and takes on those records so a commitment reported this sync is applied to them. /// 4. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the transport-delivered @@ -262,19 +268,20 @@ where self.ensure_genesis_in_place().await?; self.ensure_rpc_limits_in_place().await?; - let (note_transport_data, (state_sync, mut chain_sync_data)) = - futures::try_join!(self.fetch_note_transport_sync_data(), self.fetch_chain_updates())?; - - let transport_delivered_notes: Vec = - note_transport_data.input_note_records().cloned().collect(); + let state_sync = self.state_sync(); + let (note_transport_data, mut chain_sync_data) = futures::try_join!( + self.fetch_note_transport_sync_data(), + self.fetch_chain_updates(&state_sync) + )?; // These must be in the store before the chain data is screened: the screener // recognises a note by looking it up there, and a private note it cannot find is discarded // for good, since the chain's note query never revisits a block range. - let new_private_notes = self.apply_note_transport_sync_data(note_transport_data).await?; + let (new_private_notes, transport_delivered_notes) = + self.apply_note_transport_sync_data(note_transport_data).await?; state_sync - .process_fetched_state(&mut chain_sync_data, transport_delivered_notes) + .screen_fetched_notes(&mut chain_sync_data, transport_delivered_notes) .await?; state_sync.fetch_nullifiers(&mut chain_sync_data).await?; diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 581b194e57..deb7d524c3 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -273,8 +273,9 @@ impl StateSync { /// mutable reference so callers can keep it in memory across syncs; it is only modified once /// every check has passed. /// - /// Runs the three phases in order, each of which can also be driven separately: + /// Runs the four phases in order, each of which can also be driven separately: /// 1. [`Self::fetch_state`] — every node call but the nullifier check. + /// 2. [`Self::screen_fetched_notes`] — decide which fetched notes are relevant. /// 3. [`Self::fetch_nullifiers`] — the nullifier check. /// 4. [`Self::build_update`] — verify against the MMR and assemble the update. pub async fn sync_state( @@ -285,7 +286,7 @@ impl StateSync { let block_num = block_num_from_forest(current_partial_mmr)?; let mut chain_sync_data = self.fetch_state(block_num, input).await?; - self.process_fetched_state(&mut chain_sync_data, Vec::new()).await?; + self.screen_fetched_notes(&mut chain_sync_data, Vec::new()).await?; self.fetch_nullifiers(&mut chain_sync_data).await?; // Work on a clone so any validation failure leaves `current_partial_mmr` untouched. @@ -303,7 +304,7 @@ impl StateSync { /// 1. Fetch sync data from the node (MMR delta, note inclusions, transactions). /// 2. Update account states (fetch updated public accounts, flag mismatched private ones). /// - /// Interpreting the response is [`Self::process_fetched_state`]'s, and the nullifier check + /// Interpreting the response is [`Self::screen_fetched_notes`]'s, and the nullifier check /// [`Self::fetch_nullifiers`]'s. Both run afterwards so a caller syncing more than one source /// can write the other source first, and check nullifiers once across all of them. pub async fn fetch_state( @@ -394,7 +395,7 @@ impl StateSync { /// [`NoteScreener::on_note_received`](crate::note::NoteScreener) recognises a note by looking /// it up in the store, and a private note it cannot find is discarded — permanently, since the /// chain's note query never revisits a block range. - pub async fn process_fetched_state( + pub async fn screen_fetched_notes( &self, chain_sync_data: &mut ChainSyncData, transport_delivered_notes: Vec, @@ -505,7 +506,7 @@ impl StateSync { /// Checks the node for nullifiers of every note `chain_sync_data` could have consumed. /// /// The query covers every note the sync tracks, including the ones - /// [`Self::process_fetched_state`] took from another sync path — so a transport-delivered note + /// [`Self::screen_fetched_notes`] took from another sync path — so a transport-delivered note /// consumed within one sync is reported as consumed by that same sync. /// /// No-op when the nullifier sync is disabled (see [`Self::disable_nullifier_sync`]) or when @@ -1438,11 +1439,11 @@ struct ChainAdvance { chain_tip_header: BlockHeader, /// MMR delta from `block_from` to the chain tip, excluding the chain-tip leaf. mmr_delta: MmrDelta, - /// Note blocks as the node returned them. [`StateSync::process_fetched_state`] drains these + /// Note blocks as the node returned them. [`StateSync::screen_fetched_notes`] drains these /// into `relevant_note_blocks`, so this is empty by the time the update is built. note_blocks_awaiting_screening: Vec, /// Transaction records as the node returned them, read by - /// [`StateSync::process_fetched_state`]. + /// [`StateSync::screen_fetched_notes`]. transactions: Vec, /// Screened blocks holding a client-relevant note, each with its `sync_notes` MMR path. relevant_note_blocks: Vec, From bffcc4d2aabf41cee4c55c9279cffee73551c1f8 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sun, 30 Aug 2026 20:05:42 -0300 Subject: [PATCH 25/43] refactor(rust-client): rename screen_fetched_notes to derive_note_and_transaction_updates --- crates/rust-client/src/sync/mod.rs | 27 ++++++----- crates/rust-client/src/sync/state_sync.rs | 56 +++++++++++------------ 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index ab4c73c740..f8250ce541 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -141,8 +141,7 @@ where let state_sync = self.state_sync(); let mut chain_sync_data = self.fetch_chain_updates(&state_sync).await?; - // No other sync path ran, so there are no transport-delivered notes to take on. - state_sync.screen_fetched_notes(&mut chain_sync_data, Vec::new()).await?; + state_sync.derive_note_and_transaction_updates(&mut chain_sync_data).await?; state_sync.fetch_nullifiers(&mut chain_sync_data).await?; self.apply_chain_updates(&state_sync, chain_sync_data).await @@ -246,10 +245,10 @@ where /// /// 1. Concurrently: the note transport fetch and [`Client::fetch_chain_updates`]. Only node and /// NTL calls happen here, which is all that benefits from overlapping. - /// 2. The transport writes. - /// 3. [`StateSync::screen_fetched_notes`], which screens the node's notes against the store — - /// hence after step 2, so a transport-delivered note is recognised rather than discarded — - /// and takes on those records so a commitment reported this sync is applied to them. + /// 2. The transport writes, whose records are then tracked in the chain sync's note updates. + /// 3. [`StateSync::derive_note_and_transaction_updates`], which screens the node's notes + /// against the store — hence after step 2, so a transport-delivered note is recognised + /// rather than discarded — and applies a commitment reported this sync to those records. /// 4. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the transport-delivered /// ones, so a note delivered and consumed in the same window is reported as consumed by this /// call. @@ -274,15 +273,19 @@ where self.fetch_chain_updates(&state_sync) )?; - // These must be in the store before the chain data is screened: the screener - // recognises a note by looking it up there, and a private note it cannot find is discarded - // for good, since the chain's note query never revisits a block range. + // The NTL notes must be in the store before the chain data is screened: the screener + // recognises a note by looking it up in the store, and the updates for private notes + // it cannot find are discarded. let (new_private_notes, transport_delivered_notes) = self.apply_note_transport_sync_data(note_transport_data).await?; - state_sync - .screen_fetched_notes(&mut chain_sync_data, transport_delivered_notes) - .await?; + // The chain tracker was built from a store snapshot taken before those writes, so the + // records the screener is about to vote on have to be added to it explicitly. + chain_sync_data + .note_updates + .track_existing_input_notes(transport_delivered_notes); + + state_sync.derive_note_and_transaction_updates(&mut chain_sync_data).await?; state_sync.fetch_nullifiers(&mut chain_sync_data).await?; let mut summary = self.apply_chain_updates(&state_sync, chain_sync_data).await?; diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index deb7d524c3..2119ff7668 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -275,7 +275,7 @@ impl StateSync { /// /// Runs the four phases in order, each of which can also be driven separately: /// 1. [`Self::fetch_state`] — every node call but the nullifier check. - /// 2. [`Self::screen_fetched_notes`] — decide which fetched notes are relevant. + /// 2. [`Self::derive_note_and_transaction_updates`] — screen the notes, apply the transactions. /// 3. [`Self::fetch_nullifiers`] — the nullifier check. /// 4. [`Self::build_update`] — verify against the MMR and assemble the update. pub async fn sync_state( @@ -286,7 +286,7 @@ impl StateSync { let block_num = block_num_from_forest(current_partial_mmr)?; let mut chain_sync_data = self.fetch_state(block_num, input).await?; - self.screen_fetched_notes(&mut chain_sync_data, Vec::new()).await?; + self.derive_note_and_transaction_updates(&mut chain_sync_data).await?; self.fetch_nullifiers(&mut chain_sync_data).await?; // Work on a clone so any validation failure leaves `current_partial_mmr` untouched. @@ -304,9 +304,10 @@ impl StateSync { /// 1. Fetch sync data from the node (MMR delta, note inclusions, transactions). /// 2. Update account states (fetch updated public accounts, flag mismatched private ones). /// - /// Interpreting the response is [`Self::screen_fetched_notes`]'s, and the nullifier check - /// [`Self::fetch_nullifiers`]'s. Both run afterwards so a caller syncing more than one source - /// can write the other source first, and check nullifiers once across all of them. + /// Interpreting the response is [`Self::derive_note_and_transaction_updates`]'s, and the + /// nullifier check [`Self::fetch_nullifiers`]'s. Both run afterwards so a caller syncing + /// more than one source can write the other source first, and check nullifiers once across + /// all of them. pub async fn fetch_state( &self, block_from: BlockNumber, @@ -379,35 +380,27 @@ impl StateSync { /// Turns the node's raw response into note and transaction updates. /// - /// Screens the received notes, applies the transaction inclusions, and recovers the public - /// notes the tracked accounts consumed. Only the last of those makes a node call; the rest is - /// store reads and local execution, which is why this is split from [`Self::fetch_state`] and - /// runs afterwards rather than concurrently. - /// - /// `transport_delivered_notes` are the notes another sync path fetched from the Note Transport - /// Layer in the same call and has already written. They are tracked here so the screener's - /// verdict has a record to apply itself to, exactly as if they had been in the store when the - /// sync input was built. + /// Screens the received notes for relevance, applies the transaction inclusions, and recovers + /// the public notes the tracked accounts consumed. Only the last of those makes a node call; + /// the rest is store reads and local execution, which is why this is split from + /// [`Self::fetch_state`] and runs afterwards rather than concurrently. /// /// # Ordering /// - /// The caller must have written `transport_delivered_notes` to the store first. + /// A note another sync path wrote in the same call must be in the store *and* in + /// `chain_sync_data`'s note updates before this runs. /// [`NoteScreener::on_note_received`](crate::note::NoteScreener) recognises a note by looking - /// it up in the store, and a private note it cannot find is discarded — permanently, since the - /// chain's note query never revisits a block range. - pub async fn screen_fetched_notes( + /// it up in the store — a private note it cannot find is discarded, permanently, since the + /// chain's note query never revisits a block range — and its verdict is then applied to the + /// tracked record, which the store lookup does not provide. + pub async fn derive_note_and_transaction_updates( &self, chain_sync_data: &mut ChainSyncData, - transport_delivered_notes: Vec, ) -> Result<(), ClientError> { let Some(advance) = chain_sync_data.advance.as_mut() else { return Ok(()); }; - chain_sync_data - .note_updates - .track_existing_input_notes(transport_delivered_notes); - advance.relevant_note_blocks = self .screen_note_blocks( core::mem::take(&mut advance.note_blocks_awaiting_screening), @@ -505,9 +498,9 @@ impl StateSync { /// Checks the node for nullifiers of every note `chain_sync_data` could have consumed. /// - /// The query covers every note the sync tracks, including the ones - /// [`Self::screen_fetched_notes`] took from another sync path — so a transport-delivered note - /// consumed within one sync is reported as consumed by that same sync. + /// The query covers every note the tracker holds, including any the caller added from another + /// sync path — so a transport-delivered note consumed within one sync is reported as consumed + /// by that same sync. /// /// No-op when the nullifier sync is disabled (see [`Self::disable_nullifier_sync`]) or when /// the node reported no progress, since there is no block range to query. @@ -1428,7 +1421,9 @@ pub struct ChainSyncData { /// What the node reported beyond `block_from`, or `None` when the client was already at the /// chain tip. advance: Option, - note_updates: NoteUpdateTracker, + /// Notes as the sync found them. A caller that wrote notes of its own after the sync input + /// was built has to track them here, or this sync's verdicts have no record to apply to. + pub(crate) note_updates: NoteUpdateTracker, transaction_updates: TransactionUpdateTracker, account_updates: AccountUpdates, } @@ -1439,11 +1434,12 @@ struct ChainAdvance { chain_tip_header: BlockHeader, /// MMR delta from `block_from` to the chain tip, excluding the chain-tip leaf. mmr_delta: MmrDelta, - /// Note blocks as the node returned them. [`StateSync::screen_fetched_notes`] drains these - /// into `relevant_note_blocks`, so this is empty by the time the update is built. + /// Note blocks as the node returned them. [`StateSync::derive_note_and_transaction_updates`] + /// drains these into `relevant_note_blocks`, so this is empty by the time the update is + /// built. note_blocks_awaiting_screening: Vec, /// Transaction records as the node returned them, read by - /// [`StateSync::screen_fetched_notes`]. + /// [`StateSync::derive_note_and_transaction_updates`]. transactions: Vec, /// Screened blocks holding a client-relevant note, each with its `sync_notes` MMR path. relevant_note_blocks: Vec, From b442c60792645fa93172dd952b1eaa4d54e8a007 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sun, 30 Aug 2026 20:22:01 -0300 Subject: [PATCH 26/43] docs(rust-client): explain the NTL note merge and nullifier coverage in sync_state --- crates/rust-client/src/sync/mod.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index f8250ce541..bd3eb467f5 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -279,13 +279,17 @@ where let (new_private_notes, transport_delivered_notes) = self.apply_note_transport_sync_data(note_transport_data).await?; - // The chain tracker was built from a store snapshot taken before those writes, so the - // records the screener is about to vote on have to be added to it explicitly. + // Merge the NTL notes into the chain `note_updates`, so a commitment the chain reported + // for one of them is applied to its record. The tracker was built before the writes + // above, so without this the screener's verdict would have no record to apply to. chain_sync_data .note_updates .track_existing_input_notes(transport_delivered_notes); state_sync.derive_note_and_transaction_updates(&mut chain_sync_data).await?; + + // Now that the NTL notes are tracked, the nullifier check covers them too: one delivered + // and consumed within this window is reported as consumed by this same sync. state_sync.fetch_nullifiers(&mut chain_sync_data).await?; let mut summary = self.apply_chain_updates(&state_sync, chain_sync_data).await?; From 7c11dc37b57e08b09e15c2db3d3b0b5fdad4ab5a Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sun, 30 Aug 2026 20:28:02 -0300 Subject: [PATCH 27/43] docs(rust-client): simplify the nullifier fetch comment in sync_state --- crates/rust-client/src/sync/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index bd3eb467f5..48014597c8 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -288,8 +288,7 @@ where state_sync.derive_note_and_transaction_updates(&mut chain_sync_data).await?; - // Now that the NTL notes are tracked, the nullifier check covers them too: one delivered - // and consumed within this window is reported as consumed by this same sync. + // Checks nullifiers both for notes fetched from the chain and from the NTL state_sync.fetch_nullifiers(&mut chain_sync_data).await?; let mut summary = self.apply_chain_updates(&state_sync, chain_sync_data).await?; From 9f9b36501ee9a4250c1838e134d954087fc3167e Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sun, 30 Aug 2026 22:13:55 -0300 Subject: [PATCH 28/43] refactor(rust-client): take note blocks from the sync response instead of refetching them --- crates/rust-client/src/note/import.rs | 240 +++++++------------ crates/rust-client/src/note/mod.rs | 2 +- crates/rust-client/src/note_transport/mod.rs | 11 +- crates/rust-client/src/sync/block_header.rs | 10 +- 4 files changed, 91 insertions(+), 172 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index d32500bc8e..b3516a65c5 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -12,8 +12,7 @@ use alloc::collections::{BTreeMap, BTreeSet}; use alloc::string::ToString; use alloc::vec::Vec; -use miden_protocol::block::{BlockHeader, BlockNumber}; -use miden_protocol::crypto::merkle::MerklePath; +use miden_protocol::block::BlockNumber; use miden_protocol::note::{ Note, NoteAttachments, @@ -26,10 +25,15 @@ use miden_protocol::note::{ use miden_standards::note::NoteFile; use miden_tx::auth::TransactionAuthenticator; -use crate::rpc::domain::note::{FetchedNote, ResolvedNoteContent, SyncedNote}; +use crate::rpc::domain::note::{ + FetchedNote, + ResolvedNoteContent, + ResolvedSyncNotesBlock, + SyncedNote, +}; use crate::rpc::{NoteContentFetch, RpcError}; use crate::store::input_note_states::ExpectedNoteState; -use crate::store::{InputNoteRecord, InputNoteState, NoteFilter, StoreError}; +use crate::store::{InputNoteRecord, InputNoteState, NoteFilter}; use crate::sync::NoteTagRecord; use crate::{Client, ClientError}; @@ -338,18 +342,17 @@ where /// Builds a note record list from note details. If a note with the same ID was already stored /// it is passed via `previous_note` so it can be updated. /// - /// Only records that need to be stored are returned: notes the node has not reported as - /// committed keep (or get) their expected record, while committed notes are returned only if - /// the new information changed them. + /// The records are returned for the caller to store. The blocks proving the inclusion of the + /// committed ones are written here, so no record can be stored as committed before the header + /// that verifies it. async fn import_note_records_by_details( &mut self, requested_notes: Vec<(Option, NoteDetails, BlockNumber, NoteTag)>, ) -> Result, ClientError> { - let mut note_updates = self.fetch_transport_notes_onchain_state(requested_notes).await?; - self.fetch_note_blocks(&mut note_updates).await?; + let note_updates = self.fetch_transport_notes_onchain_state(requested_notes).await?; let mut partial_mmr = self.get_current_partial_mmr().await?; - self.insert_note_blocks(note_updates.blocks_to_insert, &mut partial_mmr).await?; + self.insert_note_blocks(note_updates.note_blocks, &mut partial_mmr).await?; // Cache MMR so pruning can reuse in-memory MMR. self.cache_partial_mmr(partial_mmr).await?; @@ -364,26 +367,52 @@ where // -------------------------------------------------------------------------------------------- /// Fetches the on-chain state of transport-delivered notes, returning the records to write - /// and the blocks that still have to be resolved by [`Client::fetch_note_blocks`]. + /// and the blocks that committed them. /// /// A note with a stored version is passed via `previous_note` so it can be updated. Notes the - /// node has not reported as committed keep (or get) their expected record; the rest are left - /// `Unverified`, carrying the inclusion proof but not the header that verifies it. + /// node has not reported as committed keep (or get) their expected record; the rest become + /// `Committed`, since the response carries the block header that verifies their inclusion. pub(crate) async fn fetch_transport_notes_onchain_state( &self, requested_notes: Vec<(Option, NoteDetails, BlockNumber, NoteTag)>, ) -> Result { let mut lowest_request_block: BlockNumber = u32::MAX.into(); - let mut note_requests = vec![]; + let mut sync_tags = BTreeSet::new(); + let mut requested_commitments = Vec::with_capacity(requested_notes.len()); for (_, details, after_block_num, tag) in &requested_notes { - note_requests.push((details.commitment(), *tag)); + sync_tags.insert(*tag); + requested_commitments.push(details.commitment()); lowest_request_block = lowest_request_block.min(*after_block_num); } - let mut committed_notes_data = - self.sync_expected_notes(lowest_request_block, note_requests).await?; + let blocks = self.sync_expected_notes(lowest_request_block, &sync_tags).await?; let mut note_updates = TransportNoteUpdates::default(); + // An expected note has no metadata and thus no `NoteId`, so each returned note is matched + // to its request by rebuilding the id from the committed metadata. Only the blocks holding + // a match are kept: the rest hold notes under the same tag that answer no request. + let mut committed_notes_data = BTreeMap::new(); + for block in blocks { + let mut block_matched = false; + for (note_id, sync_note) in &block.notes { + let metadata = sync_note.committed.metadata(); + let Some(commitment) = requested_commitments + .iter() + .find(|commitment| NoteId::new(**commitment, metadata) == *note_id) + else { + continue; + }; + + committed_notes_data + .insert(*commitment, (sync_note.clone(), block.block_header.clone())); + block_matched = true; + } + + if block_matched { + note_updates.note_blocks.push(block); + } + } + for (previous_note, details, after_block_num, tag) in requested_notes { let mut note_record = previous_note.unwrap_or_else(|| { InputNoteRecord::new( @@ -400,7 +429,7 @@ where }); // Notes the node has not reported as committed keep their expected record untouched. - let Some(SyncedNote { committed: committed_note, content }) = + let Some((SyncedNote { committed: committed_note, content }, block_header)) = committed_notes_data.remove(¬e_record.details_commitment()) else { note_updates.notes_to_write.push(note_record); @@ -411,8 +440,7 @@ where .map(ResolvedNoteContent::into_attachments) .filter(|attachments| !attachments.is_empty()); - // Leaves the record `Unverified`: it now carries the inclusion proof and metadata but - // not the block header that verifies them. `fetch_note_blocks` finishes it. + let committed_tag = committed_note.metadata().tag(); note_record.inclusion_proof_received( committed_note.inclusion_proof().clone(), *committed_note.metadata(), @@ -422,99 +450,32 @@ where note_record.attachments_received(attachments); } - note_updates.notes_to_write.push(note_record); - } - - Ok(note_updates) - } - - /// Fetches the header and MMR proof of every block that committed one of the notes in - /// `note_updates`, then finishes the records that were waiting on them. - /// - /// Each block is fetched once even when it committed several notes. A block the client's - /// partial MMR already tracks has its header read from the store and needs no insert, so it is - /// absent from `blocks_to_insert`. - /// - /// Writes nothing and does not modify the MMR: tracking the fetched headers and storing them - /// is [`Client::insert_note_blocks`]'s job. Whether the notes it commits have already been - /// spent is [`Client::fetch_note_nullifiers`]'s. - pub(crate) async fn fetch_note_blocks( - &self, - note_updates: &mut TransportNoteUpdates, - ) -> Result<(), ClientError> { - // A record left `Unverified` by `fetch_transport_notes_onchain_state` is one the node - // reported as committed: it holds the inclusion proof but not the header verifying it. - let requested_blocks: BTreeSet = - note_updates.notes_to_write.iter().filter_map(awaiting_block_header).collect(); - - if requested_blocks.is_empty() { - return Ok(()); - } - - let partial_mmr = self.get_current_partial_mmr().await?; - - let mut block_headers = BTreeMap::new(); - for block_num in requested_blocks { - if partial_mmr.is_tracked(block_num.as_usize()) { - let (block_header, _) = self - .store - .get_block_header_by_num(block_num) - .await? - .ok_or(StoreError::BlockHeaderNotFound(block_num))?; - block_headers.insert(block_num, block_header); - continue; - } - - let (block_header, mmr_proof) = - self.rpc_api.get_block_header_with_proof(block_num).await?; - block_headers.insert(block_num, block_header.clone()); - note_updates.blocks_to_insert.insert( - block_num, - NoteBlockToInsert { - block_header, - mmr_path: mmr_proof.merkle_path().clone(), - }, - ); - } - - for note_record in &mut note_updates.notes_to_write { - let Some(block_num) = awaiting_block_header(note_record) else { - continue; - }; - let block_header = block_headers - .get(&block_num) - .expect("every committed note's block was fetched above"); - - // Read before the transition, which moves the record out of the state holding these. - let committed_tag = note_record - .metadata() - .expect("a note awaiting its block header carries metadata") - .tag(); - let details_commitment = note_record.details_commitment(); - // Once committed, the note no longer needs its expected-note tag. - if note_record.block_header_received(block_header)? { - note_updates - .tags_to_remove - .push(NoteTagRecord::with_note_source(committed_tag, details_commitment)); + if note_record.block_header_received(&block_header)? { + note_updates.tags_to_remove.push(NoteTagRecord::with_note_source( + committed_tag, + note_record.details_commitment(), + )); } + + note_updates.notes_to_write.push(note_record); } - Ok(()) + Ok(note_updates) } /// Fetches the nullifier commit heights of the notes in `note_updates`, marking those already /// spent as consumed. /// - /// Must run after [`Client::fetch_note_blocks`]: only a note the node reported as committed - /// has the metadata a nullifier is derived from, so before that there is nothing to ask - /// about. + /// Must run after [`Client::fetch_transport_notes_onchain_state`]: only a note the node + /// reported as committed has the metadata a nullifier is derived from, so before that there + /// is nothing to ask about. pub(crate) async fn fetch_note_nullifiers( &self, note_updates: &mut TransportNoteUpdates, ) -> Result<(), ClientError> { - // A record here carries a nullifier only if this batch just committed it: one that was - // already committed is dropped by `fetch_note_blocks` as unchanged. + // A record carries a nullifier only once the node has reported it committed, which is + // what supplies the metadata the nullifier is derived from. let mut nullifiers = BTreeSet::new(); let mut lowest_commitment_block: BlockNumber = u32::MAX.into(); for note_record in ¬e_updates.notes_to_write { @@ -552,14 +513,13 @@ where /// Applies the changes from `note_updates` to the store, returning the written records. /// /// The block headers go in first, so a record is never persisted as committed before the - /// header proving its inclusion. [`Client::fetch_note_blocks`] must have run beforehand, or a - /// note the node committed is written as unverified and stays that way until a later sync. + /// header proving its inclusion is tracked and stored. pub(crate) async fn apply_note_transport_updates( &mut self, note_updates: TransportNoteUpdates, ) -> Result, ClientError> { let mut partial_mmr = self.get_current_partial_mmr().await?; - self.insert_note_blocks(note_updates.blocks_to_insert, &mut partial_mmr).await?; + self.insert_note_blocks(note_updates.note_blocks, &mut partial_mmr).await?; // Cache MMR so pruning can reuse in-memory MMR. self.cache_partial_mmr(partial_mmr).await?; @@ -582,59 +542,39 @@ where Ok(note_updates.notes_to_write) } - /// Checks whether the expected notes (identified by their details commitments and tags) have - /// been committed on chain between `request_block_num` and the current block, returning the - /// matching synced notes keyed by details commitment. + /// Fetches every block between `request_block_num` and the client's sync height that holds a + /// note under one of `sync_tags`. /// - /// Expected notes have no metadata and thus no `NoteId`, so each committed note is matched by - /// reconstructing the id from the committed metadata: `NoteId::new(details_commitment, - /// metadata)`. + /// Each block carries its header and the MMR path proving its inclusion at the sync height, + /// which is the forest the client's partial MMR is at, so a note found here needs no further + /// block lookup. Deciding which of the returned notes answer a request is the caller's. async fn sync_expected_notes( &self, request_block_num: BlockNumber, - // Expected notes' details commitments with their tags. - expected_notes: Vec<(NoteDetailsCommitment, NoteTag)>, - ) -> Result, ClientError> { - let sync_tags: BTreeSet = expected_notes.iter().map(|(_, tag)| *tag).collect(); - - let mut matched_notes = BTreeMap::new(); + sync_tags: &BTreeSet, + ) -> Result, ClientError> { let current_block_num = self.get_sync_height().await?; // Notes expected only after a block we have not reached can't be committed within our // synced view yet: skip the lookup and let them stay expected until a future sync. if request_block_num > current_block_num { - return Ok(matched_notes); + return Ok(Vec::new()); } - let blocks = self + let mut blocks = self .rpc_api .sync_notes_with_content( request_block_num, current_block_num, - &sync_tags, + sync_tags, NoteContentFetch::AttachmentsOnly, ) .await .map_err(ClientError::RpcError)?; - for block in blocks { - if block.block_header.block_num() > current_block_num { - break; - } + blocks.retain(|block| block.block_header.block_num() <= current_block_num); - for sync_note in block.notes.into_values() { - let committed = &sync_note.committed; - let Some((commitment, _)) = expected_notes.iter().find(|(commitment, _)| { - NoteId::new(*commitment, committed.metadata()) == *committed.note_id() - }) else { - continue; - }; - - matched_notes.insert(*commitment, sync_note); - } - } - - Ok(matched_notes) + Ok(blocks) } } @@ -645,26 +585,15 @@ where /// that must be stored before their corresponding committed notes. #[derive(Default)] pub(crate) struct TransportNoteUpdates { - /// The records to write. A note the node has not committed stays `Expected`; one it has - /// committed is `Unverified` until [`Client::fetch_note_blocks`] supplies its block header. + /// The note records to write to the storage. notes_to_write: Vec, - /// Blocks that must be tracked and stored before the committed notes that need them, keyed by - /// block number so a block committing several notes is stored once. Filled by - /// [`Client::fetch_note_blocks`]; blocks the client already tracks are absent. - blocks_to_insert: BTreeMap, + /// Blocks holding a committed note, as the node returned them. They must be tracked and + /// stored before the note records that need them. + note_blocks: Vec, /// Note-source tags to remove, one per committed note. tags_to_remove: Vec, } -/// A block header and the MMR proof path the node returned with it, as received. -/// -/// The path is verified against the client's peaks when the block is tracked, in -/// [`Client::insert_note_blocks`]. -pub(crate) struct NoteBlockToInsert { - pub(crate) block_header: BlockHeader, - pub(crate) mmr_path: MerklePath, -} - impl TransportNoteUpdates { /// Appends another batch to this one. /// @@ -672,7 +601,7 @@ impl TransportNoteUpdates { /// resolve to the version fetched last. pub(crate) fn merge(&mut self, other: Self) { self.notes_to_write.extend(other.notes_to_write); - self.blocks_to_insert.extend(other.blocks_to_insert); + self.note_blocks.extend(other.note_blocks); self.tags_to_remove.extend(other.tags_to_remove); } } @@ -680,15 +609,6 @@ impl TransportNoteUpdates { // HELPERS // ================================================================================================ -/// The block that committed a note whose record is still awaiting its header, or `None` for any -/// other record. -fn awaiting_block_header(note_record: &InputNoteRecord) -> Option { - match note_record.state() { - InputNoteState::Unverified(state) => Some(state.inclusion_proof.location().block_num()), - _ => None, - } -} - /// Returns an error if the already-stored note is currently being processed by a local /// transaction, since an in-flight note can't be overwritten by an import. pub(crate) fn ensure_not_processing( diff --git a/crates/rust-client/src/note/mod.rs b/crates/rust-client/src/note/mod.rs index be92bff56b..bd57366f58 100644 --- a/crates/rust-client/src/note/mod.rs +++ b/crates/rust-client/src/note/mod.rs @@ -69,7 +69,7 @@ use crate::store::{InputNoteRecord, NoteFilter, OutputNoteRecord}; use crate::{Client, ClientError, IdPrefixFetchError}; mod import; -pub(crate) use import::{NoteBlockToInsert, TransportNoteUpdates, ensure_not_processing}; +pub(crate) use import::{TransportNoteUpdates, ensure_not_processing}; mod note_reader; mod note_screener; mod note_update_tracker; diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index be28948466..02e8927098 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -359,7 +359,6 @@ where let (mut note_updates, new_cursor) = self .fetch_note_transport_updates(cursor, ¬e_tags, &mut id_by_commitment) .await?; - self.fetch_note_blocks(&mut note_updates).await?; self.fetch_note_nullifiers(&mut note_updates).await?; self.apply_note_transport_updates(note_updates).await?; @@ -525,10 +524,9 @@ where /// The one write it performs is the relay outbox, which [`Client::flush_relay_outbox`] /// persists itself and which is safe to redo. /// - /// Two steps run at the end, once every page is in and the notes the node reports as committed - /// are known: [`Client::fetch_note_blocks`] resolves the blocks that committed them, and - /// [`Client::fetch_note_nullifiers`] checks whether any was already spent. Storing what they - /// produce is the apply phase's. + /// One step runs at the end, once every page is in and the notes the node reports as + /// committed are known: [`Client::fetch_note_nullifiers`] checks whether any was already + /// spent. Storing what it produces is the apply phase's. /// /// Returns empty data when note transport is not configured. pub(crate) async fn fetch_note_transport_sync_data( @@ -576,7 +574,6 @@ where // Every page is in, so the blocks that committed these notes are now known. This // finishes their records and leaves the blocks for the apply phase to store. - self.fetch_note_blocks(&mut note_transport_data.note_updates).await?; self.fetch_note_nullifiers(&mut note_transport_data.note_updates).await?; Ok(note_transport_data) @@ -631,7 +628,7 @@ where /// Everything the note transport sync is about to write, with nothing written yet. /// /// Built by [`Client::fetch_note_transport_sync_data`], which also completes it with -/// [`Client::fetch_note_blocks`] and [`Client::fetch_note_nullifiers`], and written by +/// [`Client::fetch_note_nullifiers`], and written by /// [`Client::apply_note_transport_sync_data`]. #[derive(Default)] pub(crate) struct NoteTransportSyncData { diff --git a/crates/rust-client/src/sync/block_header.rs b/crates/rust-client/src/sync/block_header.rs index 7ff1bd2e0c..3a7587e4aa 100644 --- a/crates/rust-client/src/sync/block_header.rs +++ b/crates/rust-client/src/sync/block_header.rs @@ -1,4 +1,3 @@ -use alloc::collections::BTreeMap; use alloc::sync::Arc; use alloc::vec::Vec; @@ -9,8 +8,8 @@ use miden_protocol::crypto::merkle::mmr::{Forest, InOrderIndex, PartialMmr}; use miden_protocol::{Felt, Word}; use tracing::warn; -use crate::note::NoteBlockToInsert; use crate::rpc::NodeRpcClient; +use crate::rpc::domain::note::ResolvedSyncNotesBlock; use crate::store::{BlockRelevance, StoreError}; #[cfg(feature = "testing")] use crate::test_utils::mock::MockRpcApi; @@ -123,11 +122,14 @@ impl Client { /// authentication nodes that tracking produced. pub(crate) async fn insert_note_blocks( &mut self, - blocks: BTreeMap, + blocks: Vec, partial_mmr: &mut PartialMmr, ) -> Result<(), ClientError> { let mut authenticated_blocks = Vec::with_capacity(blocks.len()); - for (block_num, block) in blocks { + for block in blocks { + let block_num = block.block_header.block_num(); + // Also skips a block the loop itself just tracked, so one returned twice is stored + // once. if partial_mmr.is_tracked(block_num.as_usize()) { continue; } From 13975b580abf10116a9dd85aa4cb6afe6d86ecc8 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Sun, 30 Aug 2026 22:54:20 -0300 Subject: [PATCH 29/43] chore: improve doc comments --- crates/rust-client/src/note_transport/mod.rs | 35 ++++---------- crates/rust-client/src/sync/mod.rs | 8 +--- crates/rust-client/src/sync/state_sync.rs | 50 +++++++++----------- 3 files changed, 34 insertions(+), 59 deletions(-) diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 02e8927098..ecb86dcf83 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -434,8 +434,8 @@ where ))) } - /// Fetch one batch of notes from the note transport network for the provided tags and build - /// the records they imply, without writing anything. + /// Fetches and returns one batch of notes from the note transport layer for the provided tags + /// without applying any update to the store. /// /// The server paginates; this method issues one RPC and returns the updates together with the /// new cursor. The returned cursor equals the input cursor when the batch was empty (i.e. no @@ -514,19 +514,10 @@ where Ok((note_updates, rcursor)) } - /// Fetches what the note transport sync is about to write, writing only the relay outbox. + /// Fetches everything the note transport sync will store. /// - /// Runs the relay-outbox flush, the per-tag history backfill and the steady-state page, and - /// returns the latter two as a [`NoteTransportSyncData`] for - /// [`Client::apply_note_transport_sync_data`]. Takes `&self` so it can run concurrently with - /// the chain sync's fetch phase. - /// - /// The one write it performs is the relay outbox, which [`Client::flush_relay_outbox`] - /// persists itself and which is safe to redo. - /// - /// One step runs at the end, once every page is in and the notes the node reports as - /// committed are known: [`Client::fetch_note_nullifiers`] checks whether any was already - /// spent. Storing what it produces is the apply phase's. + /// Runs the per-tag backfill, fetches a page of notes from the Note Transport Layer, + /// and checks the nullifiers for the returned notes. /// /// Returns empty data when note transport is not configured. pub(crate) async fn fetch_note_transport_sync_data( @@ -579,17 +570,13 @@ where Ok(note_transport_data) } - /// Writes everything [`Client::fetch_note_transport_sync_data`] fetched, returning the ids of - /// the imported notes and the records it wrote. - /// - /// The records are returned so the chain sync can track them: they are in the store now, but - /// not in the snapshot its tracker was built from. + /// Saves to the storage everything [`Client::fetch_note_transport_sync_data`] fetched, + /// returning the ids of the imported notes and the records it stored. /// /// The notes are written before the covered-tag set and the cursor, so a crash between them /// re-fetches instead of skipping notes that were never written. /// - /// The block headers go in ahead of the notes that need them. The relay outbox does not: - /// [`Client::flush_relay_outbox`] persists it during the fetch. + /// The block headers are stored before the committed notes that need them. pub(crate) async fn apply_note_transport_sync_data( &mut self, note_transport_data: NoteTransportSyncData, @@ -625,11 +612,9 @@ where // NOTE TRANSPORT SYNC DATA // ================================================================================================ -/// Everything the note transport sync is about to write, with nothing written yet. +/// Everything the note transport sync is about to save to the storage. /// -/// Built by [`Client::fetch_note_transport_sync_data`], which also completes it with -/// [`Client::fetch_note_nullifiers`], and written by -/// [`Client::apply_note_transport_sync_data`]. +/// Built by [`Client::fetch_note_transport_sync_data`]. #[derive(Default)] pub(crate) struct NoteTransportSyncData { /// Covered-tag set to persist, `None` when it did not change. diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 48014597c8..d98f298fe9 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -148,7 +148,7 @@ where } /// Fetches the node's view of everything that changed since the client's chain tip, without - /// writing anything or modifying the partial MMR. + /// storing anything or modifying the partial MMR. /// /// Builds the default sync input and runs [`StateSync::fetch_state`]. The nullifier check is /// not part of this: run [`StateSync::fetch_nullifiers`] on the result before applying it, so @@ -172,7 +172,7 @@ where .with_note_observer(Arc::new(PswapChainObserver::new(self.store.clone()))) } - /// Verifies fetched chain data against the client's partial MMR and writes the resulting + /// Verifies fetched chain data against the client's partial MMR and saves the resulting /// update to the store. /// /// Also caches the partial MMR and prunes irrelevant blocks. @@ -257,10 +257,6 @@ where /// /// Fails fast on the first error. Before step 2 nothing is written but the relay outbox, which /// [`Client::flush_relay_outbox`] persists during the fetch and the next sync retries. - /// - /// One gap remains: the chain sync's note tags are read in step 1, so a tag *first* registered - /// by this call's transport import is not part of this call's `sync_notes` query. Notes under - /// such a tag are picked up by the next sync. pub async fn sync_state(&mut self) -> Result { // Both fetch phases need genesis in place, and connecting here means the two concurrent // futures never race on the RPC client's lazy connect. diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 2119ff7668..339b93c7a5 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -273,11 +273,20 @@ impl StateSync { /// mutable reference so callers can keep it in memory across syncs; it is only modified once /// every check has passed. /// - /// Runs the four phases in order, each of which can also be driven separately: - /// 1. [`Self::fetch_state`] — every node call but the nullifier check. - /// 2. [`Self::derive_note_and_transaction_updates`] — screen the notes, apply the transactions. - /// 3. [`Self::fetch_nullifiers`] — the nullifier check. - /// 4. [`Self::build_update`] — verify against the MMR and assemble the update. + /// During the sync process, the following steps are performed: + /// 1. Fetch sync data from the node (MMR delta, note inclusions, transactions). + /// 2. Update account states (fetch updated public accounts, flag mismatched private ones). + /// 3. Screen note inclusions via the configured [`OnNoteReceived`] callback. + /// 4. Process transaction inclusions (commit local txs, record external consumers, discard + /// stale/expired txs, commit output notes). + /// 5. Recover the public notes a tracked account consumed but the client never tracked. + /// 6. Detect consumed notes via nullifier sync (optional, see + /// [`Self::disable_nullifier_sync`]). + /// 7. Advance the partial MMR to the chain tip and track the screened blocks that still hold an + /// unspent note. + /// + /// Steps 1-2 are [`Self::fetch_state`], 3-5 [`Self::derive_note_and_transaction_updates`], 6 + /// [`Self::fetch_nullifiers`] and 7 [`Self::build_update`]; each can be driven separately. pub async fn sync_state( &self, current_partial_mmr: &mut PartialMmr, @@ -298,7 +307,7 @@ impl StateSync { } /// Fetches the node's view of everything that changed since `block_from`, without verifying it - /// against the client's MMR or writing anything. + /// against the client's MMR or applying any change to the store. /// /// Covers the two node calls that depend on nothing but the sync input: /// 1. Fetch sync data from the node (MMR delta, note inclusions, transactions). @@ -378,21 +387,10 @@ impl StateSync { }) } - /// Turns the node's raw response into note and transaction updates. + /// Receives a `ChainSyncData` and derives the note and transaction updates. /// /// Screens the received notes for relevance, applies the transaction inclusions, and recovers - /// the public notes the tracked accounts consumed. Only the last of those makes a node call; - /// the rest is store reads and local execution, which is why this is split from - /// [`Self::fetch_state`] and runs afterwards rather than concurrently. - /// - /// # Ordering - /// - /// A note another sync path wrote in the same call must be in the store *and* in - /// `chain_sync_data`'s note updates before this runs. - /// [`NoteScreener::on_note_received`](crate::note::NoteScreener) recognises a note by looking - /// it up in the store — a private note it cannot find is discarded, permanently, since the - /// chain's note query never revisits a block range — and its verdict is then applied to the - /// tracked record, which the store lookup does not provide. + /// the public notes the tracked accounts consumed, fetching the notes by ID from the RPC. pub async fn derive_note_and_transaction_updates( &self, chain_sync_data: &mut ChainSyncData, @@ -425,12 +423,10 @@ impl StateSync { } /// Verifies the fetched chain data against `partial_mmr` and turns it into the update to - /// persist. + /// apply to the store. /// - /// This is the chain sync's only MMR mutation: it applies the node's delta, checks the - /// resulting peaks against the chain tip header's chain commitment, and tracks the screened - /// note blocks that still hold an unspent note. It performs no I/O, so every check runs before - /// the caller's first write, and a failure leaves `partial_mmr` to be discarded by the caller. + /// It applies the node's delta, checks the resulting peaks against the chain tip header's + /// chain commitment, and tracks the screened note blocks that still hold an unspent note. pub fn build_update( chain_sync_data: ChainSyncData, partial_mmr: &mut PartialMmr, @@ -1409,12 +1405,10 @@ impl StateSync { // ================================================================================================ /// The chain data a sync fetched from the node, before any of it has been verified against the -/// client's MMR or written. +/// client's MMR or to the store. /// /// Built by [`StateSync::fetch_state`], extended by [`StateSync::fetch_nullifiers`] and turned -/// into a [`StateSyncUpdate`] by [`StateSync::build_update`]. Carries no behavior of its own: the -/// [`StateSync`] that produced it has to stay in scope until the update is applied, because the -/// note observers accumulate per-note state during the fetch and drain it in their apply hook. +/// into a [`StateSyncUpdate`] by [`StateSync::build_update`]. pub struct ChainSyncData { /// The chain tip the sync started from. pub(crate) block_from: BlockNumber, From 5a0d45c78e87bc7f04177c8d6d14f168e257434e Mon Sep 17 00:00:00 2001 From: ricomateo Date: Mon, 31 Aug 2026 00:01:18 -0300 Subject: [PATCH 30/43] fix(rust-client): return imported notes only when the fetched information changed them --- crates/rust-client/src/note/import.rs | 30 +++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index b3516a65c5..a491683d7f 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -342,9 +342,9 @@ where /// Builds a note record list from note details. If a note with the same ID was already stored /// it is passed via `previous_note` so it can be updated. /// - /// The records are returned for the caller to store. The blocks proving the inclusion of the - /// committed ones are written here, so no record can be stored as committed before the header - /// that verifies it. + /// Only records that need to be stored are returned: notes the node has not reported as + /// committed keep (or get) their expected record, while committed notes are returned only if + /// the new information changed them. async fn import_note_records_by_details( &mut self, requested_notes: Vec<(Option, NoteDetails, BlockNumber, NoteTag)>, @@ -371,7 +371,8 @@ where /// /// A note with a stored version is passed via `previous_note` so it can be updated. Notes the /// node has not reported as committed keep (or get) their expected record; the rest become - /// `Committed`, since the response carries the block header that verifies their inclusion. + /// `Committed`, since the response carries the block header that verifies their inclusion, + /// and are returned only if the new information changed them. pub(crate) async fn fetch_transport_notes_onchain_state( &self, requested_notes: Vec<(Option, NoteDetails, BlockNumber, NoteTag)>, @@ -440,25 +441,28 @@ where .map(ResolvedNoteContent::into_attachments) .filter(|attachments| !attachments.is_empty()); - let committed_tag = committed_note.metadata().tag(); - note_record.inclusion_proof_received( - committed_note.inclusion_proof().clone(), - *committed_note.metadata(), - )?; + let metadata = *committed_note.metadata(); + let mut note_changed = note_record + .inclusion_proof_received(committed_note.inclusion_proof().clone(), metadata)?; if let Some(attachments) = attachments { - note_record.attachments_received(attachments); + note_changed |= note_record.attachments_received(attachments); } + // `block_header_received` transitions the record's state, so it must always run. + note_changed |= note_record.block_header_received(&block_header)?; + // Once committed, the note no longer needs its expected-note tag. - if note_record.block_header_received(&block_header)? { + if note_changed { note_updates.tags_to_remove.push(NoteTagRecord::with_note_source( - committed_tag, + metadata.tag(), note_record.details_commitment(), )); } - note_updates.notes_to_write.push(note_record); + if note_changed { + note_updates.notes_to_write.push(note_record); + } } Ok(note_updates) From 55049406053540855ac0a10bc69c50732eef89d4 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Mon, 31 Aug 2026 00:14:12 -0300 Subject: [PATCH 31/43] chore: update changelog --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26b0f6017f..00dae12861 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,12 +32,9 @@ * [BREAKING][rust] `Client::remove_address` and `Store::remove_address` return `bool` instead of `()`, reporting whether the address was tracked. When it wasn't, `Client::remove_address` now leaves the derived note tag in place instead of running its cleanup. * [BREAKING][type][rust] Added the `TransactionRequestError::ForeignProcedureInputsTooLong` variant ([#2187](https://github.com/0xMiden/rust-sdk/pull/2187)). -* [BREAKING][behavior][rust] `Client::sync_state` now fetches the Note Transport Layer pages and the node's chain data concurrently, and writes both afterwards, instead of running a full note transport sync before the chain sync. Notes delivered over the transport are still checked for consumption in the same call — the nullifier check runs after both fetches and covers them — but the chain sync's note-tag set is now read before they are written, so a tag first registered by this call's transport import is not part of this call's `SyncNotes` query. The transport path queries the node for those notes itself, so only other notes sharing that tag wait for the next sync. -* [BREAKING][behavior][rust] The note transport sync no longer persists its progress incrementally. The imported notes, the backfill's covered-tag set and the transport cursor are all written after every page has been fetched, so a failure part way through leaves none of them written and the next sync re-fetches. Imports dedupe by note id, so a redo is harmless. The relay outbox and the block headers of committed notes are unaffected: both are still written as they are resolved, and both are safe to redo. - ### Enhancements -* [FEATURE][rust] Split both syncs into a network phase and a write phase, so every RPC call happens before the first store write and neither phase modifies the partial MMR until the writes: `Client::fetch_chain_updates` returns a `ChainSyncData` holding everything the node reported, `ChainSyncData::fetch_nullifiers` extends the nullifier check to notes another sync path delivered, and `Client::apply_chain_updates` verifies the data against the MMR and persists it. `StateSync::sync_state` is now a wrapper over the equivalent `StateSync::fetch_state` and `StateSync::build_update`. +* [rust] The chain sync and the note transport sync are each split into a fetch phase and a store phase, so every NTL and RPC call happens before the first store write. Client::sync_state runs the two fetch phases concurrently and applies both sets of updates afterwards, instead of running a full note transport sync before the chain sync ([#2453](https://github.com/0xMiden/rust-sdk/pull/2453)). * [FEATURE][rust] Added `ChainAnchor` with `Client::execute_transaction_at` and `Client::chain_anchor_for_request` to capture and execute against a pinned reference block instead of the sync height, so a transaction summary signed at one block — which binds the reference block commitment since protocol 0.16 — can be reproduced and executed later on any client ([#2421](https://github.com/0xMiden/rust-sdk/pull/2421)). * [FEATURE][rust] A client that only watches a public account now recovers notes the account consumed authenticated, even when it never tracked them by tag. During sync it reads the note references the node attaches to the account's transactions, fetches each note body by id, and surfaces it through `InputNoteReader`. Requires node `0.15.1` ([#2300](https://github.com/0xMiden/rust-sdk/pull/2300)). * [FEATURE][cli] Added a `--payback-note-type` option to `swap` so the payback note can be created as public or private (defaults to private). Public payback works without any off-band advice now that SWAP derives the payback recipient deterministically ([#2190](https://github.com/0xMiden/rust-sdk/pull/2190)). From 90ce89335ee9fb62d4e040d172d661f8f14ff7fd Mon Sep 17 00:00:00 2001 From: ricomateo Date: Tue, 1 Sep 2026 16:32:09 -0300 Subject: [PATCH 32/43] chore: move apply_superseded_account_state call to derive_state_updates --- crates/rust-client/src/sync/mod.rs | 18 +++--- crates/rust-client/src/sync/state_sync.rs | 72 ++++++++++++----------- 2 files changed, 49 insertions(+), 41 deletions(-) diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index d98f298fe9..2b802256c9 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -141,7 +141,7 @@ where let state_sync = self.state_sync(); let mut chain_sync_data = self.fetch_chain_updates(&state_sync).await?; - state_sync.derive_note_and_transaction_updates(&mut chain_sync_data).await?; + state_sync.derive_state_updates(&mut chain_sync_data).await?; state_sync.fetch_nullifiers(&mut chain_sync_data).await?; self.apply_chain_updates(&state_sync, chain_sync_data).await @@ -150,7 +150,8 @@ where /// Fetches the node's view of everything that changed since the client's chain tip, without /// storing anything or modifying the partial MMR. /// - /// Builds the default sync input and runs [`StateSync::fetch_state`]. The nullifier check is + /// Builds the default sync input and runs [`StateSync::fetch_state`]. The state updates must + /// be derived with [`StateSync::derive_state_updates`]. The nullifier check is /// not part of this: run [`StateSync::fetch_nullifiers`] on the result before applying it, so /// it can also cover transport-delivered notes another sync path fetched in the same call. pub async fn fetch_chain_updates( @@ -175,7 +176,8 @@ where /// Verifies fetched chain data against the client's partial MMR and saves the resulting /// update to the store. /// - /// Also caches the partial MMR and prunes irrelevant blocks. + /// [`StateSync::derive_state_updates`] and [`StateSync::fetch_nullifiers`] must have run on + /// the data first. Also caches the partial MMR and prunes irrelevant blocks. /// /// # Errors /// @@ -246,13 +248,13 @@ where /// 1. Concurrently: the note transport fetch and [`Client::fetch_chain_updates`]. Only node and /// NTL calls happen here, which is all that benefits from overlapping. /// 2. The transport writes, whose records are then tracked in the chain sync's note updates. - /// 3. [`StateSync::derive_note_and_transaction_updates`], which screens the node's notes - /// against the store — hence after step 2, so a transport-delivered note is recognised - /// rather than discarded — and applies a commitment reported this sync to those records. + /// 3. [`StateSync::derive_state_updates`], which screens the node's notes against the store — + /// hence after step 2, so a transport-delivered note is recognised rather than discarded — + /// and applies a commitment reported this sync to those records. /// 4. [`StateSync::fetch_nullifiers`], covering the tracked notes *and* the transport-delivered /// ones, so a note delivered and consumed in the same window is reported as consumed by this /// call. - /// 5. The chain update, written last: a nullified transport-delivered note is persisted as an + /// 5. The chain update, written last: a nullified transport-delivered note is saved as an /// update to the row step 2 inserts. /// /// Fails fast on the first error. Before step 2 nothing is written but the relay outbox, which @@ -282,7 +284,7 @@ where .note_updates .track_existing_input_notes(transport_delivered_notes); - state_sync.derive_note_and_transaction_updates(&mut chain_sync_data).await?; + state_sync.derive_state_updates(&mut chain_sync_data).await?; // Checks nullifiers both for notes fetched from the chain and from the NTL state_sync.fetch_nullifiers(&mut chain_sync_data).await?; diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index d8359b7e1f..292d3138fe 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -284,7 +284,7 @@ impl StateSync { /// 7. Advance the partial MMR to the chain tip and track the screened blocks that still hold an /// unspent note. /// - /// Steps 1-2 are [`Self::fetch_state`], 3-5 [`Self::derive_note_and_transaction_updates`], 6 + /// Steps 1-2 are [`Self::fetch_state`], 3-5 [`Self::derive_state_updates`], 6 /// [`Self::fetch_nullifiers`] and 7 [`Self::build_update`]; each can be driven separately. pub async fn sync_state( &self, @@ -294,7 +294,7 @@ impl StateSync { let block_num = block_num_from_forest(current_partial_mmr)?; let mut chain_sync_data = self.fetch_state(block_num, input).await?; - self.derive_note_and_transaction_updates(&mut chain_sync_data).await?; + self.derive_state_updates(&mut chain_sync_data).await?; self.fetch_nullifiers(&mut chain_sync_data).await?; // Work on a clone so any validation failure leaves `current_partial_mmr` untouched. @@ -305,17 +305,14 @@ impl StateSync { Ok(update) } - /// Fetches the node's view of everything that changed since `block_from`, without verifying it - /// against the client's MMR or applying any change to the store. + /// Fetches the node's view of everything that changed since `block_from`: the MMR delta, the + /// note inclusions, the transactions, and the account states. /// - /// Covers the two node calls that depend on nothing but the sync input: - /// 1. Fetch sync data from the node (MMR delta, note inclusions, transactions). - /// 2. Update account states (fetch updated public accounts, flag mismatched private ones). - /// - /// Interpreting the response is [`Self::derive_note_and_transaction_updates`]'s, and the - /// nullifier check [`Self::fetch_nullifiers`]'s. Both run afterwards so a caller syncing - /// more than one source can write the other source first, and check nullifiers once across - /// all of them. + /// Every node call that does not depend on note screening happens here, so a caller can run + /// this concurrently with another sync's fetch. Interpreting the response is + /// [`Self::derive_state_updates`]'s, and the nullifier check [`Self::fetch_nullifiers`]'s. + /// Both run afterwards so a caller syncing more than one source can write the other source + /// first, and check nullifiers once across all of them. pub async fn fetch_state( &self, block_from: BlockNumber, @@ -333,7 +330,7 @@ impl StateSync { let account_ids: Vec = accounts.iter().map(AccountHeader::id).collect(); let note_updates = NoteUpdateTracker::new(input_notes, output_notes); - let mut transaction_updates = TransactionUpdateTracker::new(uncommitted_transactions); + let transaction_updates = TransactionUpdateTracker::new(uncommitted_transactions); let mut account_updates = AccountUpdates::default(); let Some(sync_data) = self.fetch_sync_data(block_from, &account_ids, ¬e_tags).await? @@ -342,6 +339,7 @@ impl StateSync { return Ok(ChainSyncData { block_from, advance: None, + superseded_states: Vec::new(), note_updates, transaction_updates, account_updates, @@ -366,11 +364,6 @@ impl StateSync { ) .await?; - // Discard the local transactions whose result lost a same-nonce race against the network. - for superseded_state in superseded_states { - transaction_updates.apply_superseded_account_state(superseded_state); - } - Ok(ChainSyncData { block_from, advance: Some(ChainAdvance { @@ -380,43 +373,54 @@ impl StateSync { transactions, relevant_note_blocks: Vec::new(), }), + superseded_states, note_updates, transaction_updates, account_updates, }) } - /// Receives a `ChainSyncData` and derives the note and transaction updates. + /// Turns the node's raw response into note and transaction updates. /// - /// Screens the received notes for relevance, applies the transaction inclusions, and recovers - /// the public notes the tracked accounts consumed, fetching the notes by ID from the RPC. - pub async fn derive_note_and_transaction_updates( + /// Discards the local transactions the node superseded, screens the received notes for + /// relevance, applies the transaction inclusions, and recovers the public notes the tracked + /// accounts consumed, fetching those by id. + pub async fn derive_state_updates( &self, chain_sync_data: &mut ChainSyncData, ) -> Result<(), ClientError> { - let Some(advance) = chain_sync_data.advance.as_mut() else { + let ChainSyncData { + advance, + superseded_states, + note_updates, + transaction_updates, + .. + } = chain_sync_data; + + let Some(advance) = advance.as_mut() else { return Ok(()); }; + // Discard the local transactions whose result lost a same-nonce race against the network. + for superseded_state in core::mem::take(superseded_states) { + transaction_updates.apply_superseded_account_state(superseded_state); + } + advance.relevant_note_blocks = self .screen_note_blocks( core::mem::take(&mut advance.note_blocks_awaiting_screening), - &mut chain_sync_data.note_updates, + note_updates, ) .await?; self.apply_transactions_and_nullifiers( &advance.chain_tip_header, &advance.transactions, - &mut chain_sync_data.note_updates, - &mut chain_sync_data.transaction_updates, + note_updates, + transaction_updates, )?; - self.recover_consumed_public_notes( - &mut chain_sync_data.note_updates, - &advance.transactions, - ) - .await?; + self.recover_consumed_public_notes(note_updates, &advance.transactions).await?; Ok(()) } @@ -1444,6 +1448,8 @@ pub struct ChainSyncData { /// What the node reported beyond `block_from`, or `None` when the client was already at the /// chain tip. advance: Option, + /// Account states the node superseded, to be applied to the transaction updates. + superseded_states: Vec, /// Notes as the sync found them. A caller that wrote notes of its own after the sync input /// was built has to track them here, or this sync's verdicts have no record to apply to. pub(crate) note_updates: NoteUpdateTracker, @@ -1457,12 +1463,12 @@ struct ChainAdvance { chain_tip_header: BlockHeader, /// MMR delta from `block_from` to the chain tip, excluding the chain-tip leaf. mmr_delta: MmrDelta, - /// Note blocks as the node returned them. [`StateSync::derive_note_and_transaction_updates`] + /// Note blocks as the node returned them. [`StateSync::derive_state_updates`] /// drains these into `relevant_note_blocks`, so this is empty by the time the update is /// built. note_blocks_awaiting_screening: Vec, /// Transaction records as the node returned them, read by - /// [`StateSync::derive_note_and_transaction_updates`]. + /// [`StateSync::derive_state_updates`]. transactions: Vec, /// Screened blocks holding a client-relevant note, each with its `sync_notes` MMR path. relevant_note_blocks: Vec, From c987407875b1d89d01eb6143ddd7faf20fa88df7 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Tue, 1 Sep 2026 16:44:02 -0300 Subject: [PATCH 33/43] chore: add type alias for the import_note_records_by_details parameter --- crates/rust-client/src/note/import.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 925923128a..3231ff2296 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -342,7 +342,7 @@ where /// the new information changed them. async fn import_note_records_by_details( &mut self, - requested_notes: Vec<(Option, NoteDetails, BlockNumber, NoteTag)>, + requested_notes: Vec, ) -> Result, ClientError> { let note_updates = self.fetch_transport_notes_onchain_state(requested_notes).await?; @@ -370,7 +370,7 @@ where /// and are returned only if the new information changed them. pub(crate) async fn fetch_transport_notes_onchain_state( &self, - requested_notes: Vec<(Option, NoteDetails, BlockNumber, NoteTag)>, + requested_notes: Vec, ) -> Result { let mut lowest_request_block: BlockNumber = u32::MAX.into(); let mut sync_tags = BTreeSet::new(); @@ -597,6 +597,10 @@ where // EXPECTED NOTE IMPORT // ================================================================================================ +/// A note to import: the stored record it updates when there is one, its details, the block from +/// which to look for its commitment, and the tag to track it under. +pub(crate) type NoteImportRequest = (Option, NoteDetails, BlockNumber, NoteTag); + /// Notes fetched from the Note Transport Layer, along with note tags to remove, and the blocks /// that must be stored before their corresponding committed notes. #[derive(Default)] From d24f66990e283c68457d7dac7a7f83105ee33adc Mon Sep 17 00:00:00 2001 From: ricomateo Date: Fri, 4 Sep 2026 12:25:30 -0300 Subject: [PATCH 34/43] chore: replace assertion with an error --- crates/rust-client/src/errors.rs | 2 ++ crates/rust-client/src/sync/state_sync.rs | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/rust-client/src/errors.rs b/crates/rust-client/src/errors.rs index 8df676d2b5..955a14fc73 100644 --- a/crates/rust-client/src/errors.rs +++ b/crates/rust-client/src/errors.rs @@ -235,6 +235,8 @@ pub enum ClientError { /// returning `Observer(Box::new(err))`. #[error(transparent)] Observer(Box), + #[error("expected note blocks to be screened before state sync update is built")] + UnscreenedNoteBlocks, } // OBSERVER FAN-OUT diff --git a/crates/rust-client/src/sync/state_sync.rs b/crates/rust-client/src/sync/state_sync.rs index 292d3138fe..f123d31ca9 100644 --- a/crates/rust-client/src/sync/state_sync.rs +++ b/crates/rust-client/src/sync/state_sync.rs @@ -462,10 +462,10 @@ impl StateSync { account_updates, )); }; - assert!( - note_blocks_awaiting_screening.is_empty(), - "note blocks must be screened before the update is built" - ); + // Check the note blocks have been screened before building the update + if !note_blocks_awaiting_screening.is_empty() { + return Err(ClientError::UnscreenedNoteBlocks); + } let chain_tip = chain_tip_header.block_num(); From c5980b588dc694bb5b872cddd0f77eb4fcaa90f7 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Wed, 9 Sep 2026 19:21:14 -0300 Subject: [PATCH 35/43] refactor(rust-client): make only the note transport fetch concurrent with the chain fetch --- crates/rust-client/src/note/import.rs | 163 ++++++------------- crates/rust-client/src/note/mod.rs | 1 - crates/rust-client/src/note_transport/mod.rs | 128 ++++++--------- crates/rust-client/src/sync/mod.rs | 30 ++-- 4 files changed, 112 insertions(+), 210 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index e1e1929ff7..7c5558e270 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -68,6 +68,18 @@ where &mut self, note_files: &[NoteFile], ) -> Result, ClientError> { + let records = self.import_note_records(note_files).await?; + Ok(records.iter().map(InputNoteRecord::details_commitment).collect()) + } + + /// Imports `note_files` and returns the records written to the store. + /// + /// A caller that must act on the imported notes in the same operation needs the records, not + /// just their commitments: a sync uses them to extend its own note updates. + pub(crate) async fn import_note_records( + &mut self, + note_files: &[NoteFile], + ) -> Result, ClientError> { self.ensure_genesis_in_place().await?; // Deduplicate the incoming files, keeping note IDs and details commitments in separate @@ -156,20 +168,19 @@ where imported_notes.extend(notes_by_proof); } - let mut imported_commitments = Vec::with_capacity(imported_notes.len()); - for note in imported_notes { - let details_commitment = note.details_commitment(); + for note in &imported_notes { + // A record still expected needs its tag tracked so a later sync finds it. A committed + // one is no longer in that state, so it is skipped here. if let InputNoteState::Expected(ExpectedNoteState { tag: Some(tag), .. }) = note.state() { self.store - .add_note_tag(NoteTagRecord::with_note_source(*tag, details_commitment)) + .add_note_tag(NoteTagRecord::with_note_source(*tag, note.details_commitment())) .await?; } - self.store.upsert_input_notes(&[note]).await?; - imported_commitments.push(details_commitment); } + self.store.upsert_input_notes(&imported_notes).await?; - Ok(imported_commitments) + Ok(imported_notes) } // HELPERS @@ -344,34 +355,6 @@ where &mut self, requested_notes: Vec, ) -> Result, ClientError> { - let note_updates = self.fetch_transport_notes_onchain_state(requested_notes).await?; - - let mut partial_mmr = self.get_current_partial_mmr().await?; - self.insert_note_blocks(note_updates.note_blocks, &mut partial_mmr).await?; - // Cache MMR so pruning can reuse in-memory MMR. - self.cache_partial_mmr(partial_mmr).await?; - - for tag in note_updates.tags_to_remove { - self.store.remove_note_tag(tag).await?; - } - - Ok(note_updates.notes_to_write) - } - - // TRANSPORT-DELIVERED NOTE IMPORT - // -------------------------------------------------------------------------------------------- - - /// Fetches the on-chain state of transport-delivered notes, returning the records to write and - /// the blocks that committed them. - /// - /// A note with a stored version is passed via `previous_note` so it can be updated. Notes the - /// node has not reported as committed keep (or get) their expected record; the rest become - /// `Committed`, since the response carries the block header that verifies their inclusion, and - /// are returned only if the new information changed them. - pub(crate) async fn fetch_transport_notes_onchain_state( - &self, - requested_notes: Vec, - ) -> Result { let mut lowest_request_block: BlockNumber = u32::MAX.into(); let mut sync_tags = BTreeSet::new(); let mut requested_commitments = Vec::with_capacity(requested_notes.len()); @@ -382,12 +365,11 @@ where } let blocks = self.sync_expected_notes(lowest_request_block, &sync_tags).await?; - let mut note_updates = TransportNoteUpdates::default(); - // An expected note has no metadata and thus no `NoteId`, so each returned note is matched // to its request by rebuilding the id from the committed metadata. Only the blocks holding // a match are kept: the rest hold notes under the same tag that answer no request. let mut committed_notes_data = BTreeMap::new(); + let mut matched_blocks = Vec::new(); for block in blocks { let mut block_matched = false; for (note_id, sync_note) in &block.notes { @@ -405,10 +387,18 @@ where } if block_matched { - note_updates.note_blocks.push(block); + matched_blocks.push(block); } } + // The blocks arrive with the notes, so a committed note needs no further block lookup. They + // are stored first, so a record is never persisted as committed before the header that + // proves its inclusion is tracked and stored. + let mut partial_mmr = self.get_current_partial_mmr().await?; + self.insert_note_blocks(matched_blocks, &mut partial_mmr).await?; + self.cache_partial_mmr(partial_mmr).await?; + + let mut note_records = vec![]; for (previous_note, details, after_block_num, tag) in requested_notes { let mut note_record = previous_note.unwrap_or_else(|| { InputNoteRecord::new( @@ -432,7 +422,7 @@ where block_header, )) = committed_notes_data.remove(¬e_record.details_commitment()) else { - note_updates.notes_to_write.push(note_record); + note_records.push(note_record); continue; }; @@ -452,35 +442,39 @@ where // Once committed, the note no longer needs its expected-note tag. if note_changed { - note_updates.tags_to_remove.push(NoteTagRecord::with_note_source( - metadata.tag(), - note_record.details_commitment(), - )); + self.store + .remove_note_tag(NoteTagRecord::with_note_source( + metadata.tag(), + note_record.details_commitment(), + )) + .await?; } if note_changed { - note_updates.notes_to_write.push(note_record); + note_records.push(note_record); } } - Ok(note_updates) + self.mark_externally_consumed(&mut note_records).await?; + + Ok(note_records) } - /// Fetches the nullifier commit heights of the notes in `note_updates`, marking those already - /// spent as consumed. + /// Marks every record in `note_records` whose nullifier is already on chain as consumed. /// - /// Must run after [`Client::fetch_transport_notes_onchain_state`]: only a note the node - /// reported as committed has the metadata a nullifier is derived from, so before that there is - /// nothing to ask about. - pub(crate) async fn fetch_note_nullifiers( + /// The query starts at the lowest block that committed one of these notes, so it also covers a + /// note spent below the client's checkpoint. A sync only queries nullifiers from its own + /// checkpoint forward and would never revisit that block. + /// + /// Only a note the node reported as committed carries the metadata a nullifier is derived from, + /// so the rest are skipped. + async fn mark_externally_consumed( &self, - note_updates: &mut TransportNoteUpdates, + note_records: &mut [InputNoteRecord], ) -> Result<(), ClientError> { - // A record carries a nullifier only once the node has reported it committed, which is what - // supplies the metadata the nullifier is derived from. let mut nullifiers = BTreeSet::new(); let mut lowest_commitment_block: BlockNumber = u32::MAX.into(); - for note_record in ¬e_updates.notes_to_write { + for note_record in note_records.iter() { let (Some(nullifier), Some(inclusion_proof)) = (note_record.nullifier(), note_record.inclusion_proof()) else { @@ -500,7 +494,7 @@ where .get_nullifier_commit_heights(nullifiers, lowest_commitment_block) .await?; - for note_record in &mut note_updates.notes_to_write { + for note_record in note_records.iter_mut() { let Some(nullifier) = note_record.nullifier() else { continue; }; @@ -512,38 +506,6 @@ where Ok(()) } - /// Applies the changes from `note_updates` to the store, returning the written records. - /// - /// The block headers go in first, so a record is never persisted as committed before the header - /// proving its inclusion is tracked and stored. - pub(crate) async fn apply_note_transport_updates( - &mut self, - note_updates: TransportNoteUpdates, - ) -> Result, ClientError> { - let mut partial_mmr = self.get_current_partial_mmr().await?; - self.insert_note_blocks(note_updates.note_blocks, &mut partial_mmr).await?; - // Cache MMR so pruning can reuse in-memory MMR. - self.cache_partial_mmr(partial_mmr).await?; - - for tag in note_updates.tags_to_remove { - self.store.remove_note_tag(tag).await?; - } - - for note in ¬e_updates.notes_to_write { - // A record still expected needs its tag tracked so a later sync finds it. A committed - // one is no longer in that state, so it is skipped here. - if let InputNoteState::Expected(ExpectedNoteState { tag: Some(tag), .. }) = note.state() - { - self.store - .add_note_tag(NoteTagRecord::with_note_source(*tag, note.details_commitment())) - .await?; - } - } - self.store.upsert_input_notes(¬e_updates.notes_to_write).await?; - - Ok(note_updates.notes_to_write) - } - /// Fetches every block between `request_block_num` and the client's sync height that holds a /// note under one of `sync_tags`. /// @@ -601,31 +563,6 @@ where /// which to look for its commitment, and the tag to track it under. pub(crate) type NoteImportRequest = (Option, NoteDetails, BlockNumber, NoteTag); -/// Notes fetched from the Note Transport Layer, along with note tags to remove, and the blocks that -/// must be stored before their corresponding committed notes. -#[derive(Default)] -pub(crate) struct TransportNoteUpdates { - /// The note records to write to the storage. - notes_to_write: Vec, - /// Blocks holding a committed note, as the node returned them. They must be tracked and stored - /// before the note records that need them. - note_blocks: Vec, - /// Note-source tags to remove, one per committed note. - tags_to_remove: Vec, -} - -impl TransportNoteUpdates { - /// Appends another batch to this one. - /// - /// Order is preserved, which is what makes a note returned by more than one transport page - /// resolve to the version fetched last. - pub(crate) fn merge(&mut self, other: Self) { - self.notes_to_write.extend(other.notes_to_write); - self.note_blocks.extend(other.note_blocks); - self.tags_to_remove.extend(other.tags_to_remove); - } -} - // HELPERS // ================================================================================================ diff --git a/crates/rust-client/src/note/mod.rs b/crates/rust-client/src/note/mod.rs index 28cd34edb0..9f080a35d5 100644 --- a/crates/rust-client/src/note/mod.rs +++ b/crates/rust-client/src/note/mod.rs @@ -69,7 +69,6 @@ use crate::store::{InputNoteRecord, NoteFilter, OutputNoteRecord}; use crate::{Client, ClientError, IdPrefixFetchError}; mod import; -pub(crate) use import::{TransportNoteUpdates, ensure_not_processing}; mod note_reader; mod note_screener; mod note_update_tracker; diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index daae05095c..eea3c49434 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -24,7 +24,7 @@ use miden_tx::utils::serde::{ }; pub use self::errors::NoteTransportError; -use crate::note::{TransportNoteUpdates, ensure_not_processing}; +use crate::note::{NoteFile, NoteSyncHint}; use crate::store::{InputNoteRecord, NoteFilter, SettingScope}; use crate::sync::NoteTagSource; use crate::{Client, ClientError}; @@ -361,12 +361,11 @@ where let cursor = self.store.get_note_transport_cursor().await?; let mut id_by_commitment = BTreeMap::new(); - let (mut note_updates, new_cursor) = self + let (note_files, new_cursor) = self .fetch_note_transport_updates(cursor, ¬e_tags, &mut id_by_commitment) .await?; - self.fetch_note_nullifiers(&mut note_updates).await?; - self.apply_note_transport_updates(note_updates).await?; + self.import_note_records(¬e_files).await?; self.store.update_note_transport_cursor(new_cursor).await?; Ok(()) @@ -417,20 +416,20 @@ where &self, tag: NoteTag, id_by_commitment: &mut BTreeMap, - ) -> Result { - let mut note_updates = TransportNoteUpdates::default(); + ) -> Result, ClientError> { + let mut note_files = Vec::new(); let mut cursor = NoteTransportCursor::init(); for _ in 0..Self::MAX_BACKFILL_ITERATIONS { - let (page_updates, new_cursor) = + let (page_files, new_cursor) = self.fetch_note_transport_updates(cursor, &[tag], id_by_commitment).await?; - note_updates.merge(page_updates); + note_files.extend(page_files); // Terminate on any lack of forward progress. A well-behaved server returns `new_cursor // == cursor` when there are no new notes for this tag (since `rcursor = max(cursor, // max_seq_returned)`); using `<=` also handles implementations that return an `init()` // cursor on empty batches (see the in-tree mock transport). if new_cursor <= cursor { - return Ok(note_updates); + return Ok(note_files); } cursor = new_cursor; } @@ -459,7 +458,7 @@ where cursor: NoteTransportCursor, tags: &[NoteTag], id_by_commitment: &mut BTreeMap, - ) -> Result<(TransportNoteUpdates, NoteTransportCursor), ClientError> { + ) -> Result<(Vec, NoteTransportCursor), ClientError> { // Fallback lookback window, in blocks, used only for notes the transport delivered without // a sender-provided block hint. Scanning back from sync height handles the race where a // note is committed on-chain just before the NTL delivers its data. Without it, @@ -506,54 +505,38 @@ where let fallback_after_block_num = BlockNumber::from(sync_height.as_u32().saturating_sub(NOTE_LOOKBACK_BLOCKS)); - // Deduplicate by details commitment, so a note delivered twice is requested once. - let mut requests_by_commitment = BTreeMap::new(); + let mut note_files = Vec::with_capacity(notes.len()); for (note, block_hint) in notes { let tag = note.metadata().tag(); // Prefer the sender-provided hint, falling back to the lookback window when absent. let after_block_num = block_hint.unwrap_or(fallback_after_block_num); - let details = NoteDetails::from(note); - requests_by_commitment.insert(details.commitment(), (details, after_block_num, tag)); + note_files.push(NoteFile::ExpectedNote { + details: note.into(), + sync_hint: NoteSyncHint::new(after_block_num, tag), + }); } - let mut previous_by_commitment: BTreeMap = self - .get_input_notes(NoteFilter::DetailsCommitments( - requests_by_commitment.keys().copied().collect(), - )) - .await? - .into_iter() - .map(|note| (note.details_commitment(), note)) - .collect(); - - let mut requests = Vec::with_capacity(requests_by_commitment.len()); - for (commitment, (details, after_block_num, tag)) in requests_by_commitment { - let previous_note = previous_by_commitment.remove(&commitment); - ensure_not_processing(previous_note.as_ref())?; - requests.push((previous_note, details, after_block_num, tag)); - } - - let note_updates = self.fetch_transport_notes_onchain_state(requests).await?; - - Ok((note_updates, rcursor)) + Ok((note_files, rcursor)) } - /// Fetches everything the note transport sync will store. + /// Fetches the notes the Note Transport Layer holds for the tracked tags. /// - /// Runs the per-tag backfill, fetches a page of notes from the Note Transport Layer, and checks - /// the nullifiers for the returned notes. + /// Runs the per-tag backfill and fetches a page of notes. This performs no node call and writes + /// nothing but the relay outbox, so it can run concurrently with the chain fetch. The caller + /// imports the returned files and then persists the cursor and the covered-tag set. /// /// Returns empty data when note transport is not configured. - pub(crate) async fn fetch_note_transport_sync_data( + pub(crate) async fn fetch_note_transport_notes( &self, - ) -> Result { - let mut note_transport_data = NoteTransportSyncData::default(); + ) -> Result { + let mut fetch = NoteTransportFetch::default(); if !self.is_note_transport_enabled() { - return Ok(note_transport_data); + return Ok(fetch); } // Drain any private notes whose previous relay attempt failed. A flush error is logged, not // propagated: a failing relay must not block the sync, and the entries stay durable for the - // next attempt. This is the one write the fetch phase performs; it touches only the outbox + // next attempt. This is the one write this phase performs; it touches only the outbox // setting, which is independent of everything the apply phase writes. if let Err(err) = self.flush_relay_outbox().await { tracing::warn!(?err, "relay outbox flush failed during sync; entries retained"); @@ -564,54 +547,44 @@ where let (mut covered, pruned, new_tags) = self.plan_backfill().await?; let backfilled = !new_tags.is_empty(); for tag in new_tags { - note_transport_data - .note_updates - .merge(self.backfill_tag(tag, &mut note_transport_data.id_by_commitment).await?); + fetch + .note_files + .extend(self.backfill_tag(tag, &mut fetch.id_by_commitment).await?); covered.insert(tag); } if pruned || backfilled { - note_transport_data.covered_tags = Some(covered); + fetch.covered_tags = Some(covered); } let cursor = self.store.get_note_transport_cursor().await?; let note_tags: Vec = self.store.get_unique_note_tags().await?.into_iter().collect(); - let (note_updates, new_cursor) = self - .fetch_note_transport_updates( - cursor, - ¬e_tags, - &mut note_transport_data.id_by_commitment, - ) + let (note_files, new_cursor) = self + .fetch_note_transport_updates(cursor, ¬e_tags, &mut fetch.id_by_commitment) .await?; - note_transport_data.note_updates.merge(note_updates); - note_transport_data.cursor = Some(new_cursor); - - // Every page is in, so the blocks that committed these notes are now known. This finishes - // their records and leaves the blocks for the apply phase to store. - self.fetch_note_nullifiers(&mut note_transport_data.note_updates).await?; + fetch.note_files.extend(note_files); + fetch.cursor = Some(new_cursor); - Ok(note_transport_data) + Ok(fetch) } - /// Saves to the storage everything [`Client::fetch_note_transport_sync_data`] fetched, - /// returning the ids of the imported notes and the records it stored. + /// Imports what [`Client::fetch_note_transport_notes`] returned, returning the ids of the + /// imported notes and the records written. /// /// The notes are written before the covered-tag set and the cursor, so a crash between them /// re-fetches instead of skipping notes that were never written. - /// - /// The block headers are stored before the committed notes that need them. - pub(crate) async fn apply_note_transport_sync_data( + pub(crate) async fn import_note_transport_notes( &mut self, - note_transport_data: NoteTransportSyncData, + fetch: NoteTransportFetch, ) -> Result<(Vec, Vec), ClientError> { - let NoteTransportSyncData { - covered_tags, - note_updates, + let NoteTransportFetch { + note_files, id_by_commitment, + covered_tags, cursor, - } = note_transport_data; + } = fetch; - let written = self.apply_note_transport_updates(note_updates).await?; + let written = self.import_note_records(¬e_files).await?; let mut imported_ids: Vec = written .iter() .filter_map(|note| id_by_commitment.get(¬e.details_commitment()).copied()) @@ -658,21 +631,22 @@ where } } -// NOTE TRANSPORT SYNC DATA +// NOTE TRANSPORT FETCH // ================================================================================================ -/// Everything the note transport sync is about to save to the storage. +/// What the note transport fetch returned, before anything is written. /// -/// Built by [`Client::fetch_note_transport_sync_data`]. +/// Built by [`Client::fetch_note_transport_notes`] and consumed by +/// [`Client::import_note_transport_notes`]. #[derive(Default)] -pub(crate) struct NoteTransportSyncData { - /// Covered-tag set to persist, `None` when it did not change. - covered_tags: Option>, - /// Every fetched page's updates, merged in fetch order. - pub(crate) note_updates: TransportNoteUpdates, +pub(crate) struct NoteTransportFetch { + /// Notes to import, backfill pages first and then the steady-state page. + note_files: Vec, /// Note ids by details commitment, taken from the note headers the transport returned. Used to /// resolve the written records back to ids. id_by_commitment: BTreeMap, + /// Covered-tag set to persist, `None` when it did not change. + covered_tags: Option>, /// New global cursor, from the steady-state page. `None` when no page was fetched. cursor: Option, } diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index d339de08e6..5c787e1933 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -232,9 +232,9 @@ where } self.ensure_genesis_in_place().await?; - let note_transport_data = self.fetch_note_transport_sync_data().await?; + let fetch = self.fetch_note_transport_notes().await?; - let (imported_ids, _) = self.apply_note_transport_sync_data(note_transport_data).await?; + let (imported_ids, _) = self.import_note_transport_notes(fetch).await?; Ok(imported_ids) } @@ -265,31 +265,23 @@ where self.ensure_rpc_limits_in_place().await?; let state_sync = self.state_sync(); - let (note_transport_data, mut chain_sync_data) = futures::try_join!( - self.fetch_note_transport_sync_data(), - self.fetch_chain_updates(&state_sync) + let (transport_fetch, mut chain_sync_data) = futures::try_join!( + self.fetch_note_transport_notes(), + self.fetch_chain_updates(&state_sync), )?; - // The NTL notes must be in the store before the chain data is screened: the screener - // recognises a note by looking it up in the store, and the updates for private notes it - // cannot find are discarded. - let (new_private_notes, transport_delivered_notes) = - self.apply_note_transport_sync_data(note_transport_data).await?; + let (new_private_notes, imported) = + self.import_note_transport_notes(transport_fetch).await?; - // Merge the NTL notes into the chain `note_updates`, so a commitment the chain reported for - // one of them is applied to its record. The tracker was built before the writes above, so - // without this the screener's verdict would have no record to apply to. - chain_sync_data - .note_updates - .track_existing_input_notes(transport_delivered_notes); + // The chain sync built its note updates from a store snapshot taken before the import, so + // the imported records are added here. Without them this sync has no record to apply its + // verdicts to, and a note committed within this sync's own block range stays expected. + chain_sync_data.note_updates.track_existing_input_notes(imported); state_sync.derive_state_updates(&mut chain_sync_data).await?; - - // Checks nullifiers both for notes fetched from the chain and from the NTL state_sync.fetch_nullifiers(&mut chain_sync_data).await?; let mut summary = self.apply_chain_updates(&state_sync, chain_sync_data).await?; - summary.new_private_notes = new_private_notes; Ok(summary) } From d3097017dcd164b3751a945713bc24b55656aa22 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 10 Sep 2026 11:26:05 -0300 Subject: [PATCH 36/43] fix(rust-client): keep the consumer account of a note spent above the sync height --- crates/rust-client/src/note/import.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 7c5558e270..1e54ae761a 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -460,11 +460,8 @@ where Ok(note_records) } - /// Marks every record in `note_records` whose nullifier is already on chain as consumed. - /// - /// The query starts at the lowest block that committed one of these notes, so it also covers a - /// note spent below the client's checkpoint. A sync only queries nullifiers from its own - /// checkpoint forward and would never revisit that block. + /// Marks a record whose nullifier is already on chain as consumed, when the nullifier commit + /// height is at or below the client's sync height. /// /// Only a note the node reported as committed carries the metadata a nullifier is derived from, /// so the rest are skipped. @@ -494,11 +491,14 @@ where .get_nullifier_commit_heights(nullifiers, lowest_commitment_block) .await?; + let sync_height = self.get_sync_height().await?; for note_record in note_records.iter_mut() { let Some(nullifier) = note_record.nullifier() else { continue; }; - if let Some(Some(spent_at)) = spent_heights.get(&nullifier) { + if let Some(Some(spent_at)) = spent_heights.get(&nullifier) + && *spent_at <= sync_height + { note_record.consumed_externally(nullifier, *spent_at, None)?; } } From 3e5ce6fb2d43732deb882edbc2358119ad7c5727 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 10 Sep 2026 12:45:57 -0300 Subject: [PATCH 37/43] fix: replace try_join with join --- crates/rust-client/src/sync/mod.rs | 24 +++++++++++++------ .../miden-client-tests/src/tests/transport.rs | 4 ++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 5c787e1933..7c3d74635b 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -69,7 +69,7 @@ use miden_protocol::note::NoteId; use miden_protocol::transaction::TransactionId; use miden_tx::auth::TransactionAuthenticator; use miden_tx::utils::serde::{Deserializable, DeserializationError, Serializable}; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use crate::pswap::PswapChainObserver; use crate::store::{NoteFilter, TransactionFilter}; @@ -246,7 +246,8 @@ where /// /// 1. Concurrently: the note transport fetch and [`Client::fetch_chain_updates`]. Only node and /// NTL calls happen here, which is all that benefits from overlapping. - /// 2. The transport writes, whose records are then tracked in the chain sync's note updates. + /// 2. The transport writes, when its fetch succeeded, whose records are then tracked in the + /// chain sync's note updates. /// 3. [`StateSync::derive_state_updates`], which screens the node's notes against the store — /// hence after step 2, so a transport-delivered note is recognised rather than discarded — /// and applies a commitment reported this sync to those records. @@ -256,7 +257,8 @@ where /// 5. The chain update, written last: a nullified transport-delivered note is saved as an /// update to the row step 2 inserts. /// - /// Fails fast on the first error. Before step 2 nothing is written but the relay outbox, which + /// A transport failure is logged and the chain sync continues without it, leaving the transport + /// cursor for the next call to retry. Before step 2 but the relay outbox, which /// [`Client::flush_relay_outbox`] persists during the fetch and the next sync retries. pub async fn sync_state(&mut self) -> Result { // Both fetch phases need genesis in place, and connecting here means the two concurrent @@ -265,13 +267,21 @@ where self.ensure_rpc_limits_in_place().await?; let state_sync = self.state_sync(); - let (transport_fetch, mut chain_sync_data) = futures::try_join!( + let (transport_fetch, chain_sync_data) = futures::join!( self.fetch_note_transport_notes(), self.fetch_chain_updates(&state_sync), - )?; + ); + + // An NTL failure does not end the sync + let (new_private_notes, imported) = match transport_fetch { + Ok(fetch) => self.import_note_transport_notes(fetch).await?, + Err(err) => { + warn!(?err, "note transport fetch failed; syncing the chain without it"); + (Vec::new(), Vec::new()) + }, + }; - let (new_private_notes, imported) = - self.import_note_transport_notes(transport_fetch).await?; + let mut chain_sync_data = chain_sync_data?; // The chain sync built its note updates from a store snapshot taken before the import, so // the imported records are added here. Without them this sync has no record to apply its diff --git a/crates/testing/miden-client-tests/src/tests/transport.rs b/crates/testing/miden-client-tests/src/tests/transport.rs index 9f7ea2fc4a..6f659e07d1 100644 --- a/crates/testing/miden-client-tests/src/tests/transport.rs +++ b/crates/testing/miden-client-tests/src/tests/transport.rs @@ -1149,8 +1149,8 @@ async fn transport_fetch_failure_leaves_cursor_for_retry() { .add_note(*note.header(), NoteDetails::from(note.clone()).to_bytes()); faulty.fail_next_n_fetches(2); - recipient.sync_state().await.unwrap_err(); - recipient.sync_state().await.unwrap_err(); + recipient.sync_state().await.unwrap(); + recipient.sync_state().await.unwrap(); assert_eq!(faulty.fetch_attempts(), 2); assert_eq!(recipient.get_input_notes(NoteFilter::All).await.unwrap().len(), 0); From 4a1b54b78fdbee517dac389728b85100d93f2122 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 10 Sep 2026 12:52:04 -0300 Subject: [PATCH 38/43] chore: rename NoteImportRequest type alias to NoteImportByDetailsRequest --- crates/rust-client/src/note/import.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 1e54ae761a..8156bb2950 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -353,7 +353,7 @@ where /// the new information changed them. async fn import_note_records_by_details( &mut self, - requested_notes: Vec, + requested_notes: Vec, ) -> Result, ClientError> { let mut lowest_request_block: BlockNumber = u32::MAX.into(); let mut sync_tags = BTreeSet::new(); @@ -556,12 +556,10 @@ where } } -// EXPECTED NOTE IMPORT -// ================================================================================================ - -/// A note to import: the stored record it updates when there is one, its details, the block from -/// which to look for its commitment, and the tag to track it under. -pub(crate) type NoteImportRequest = (Option, NoteDetails, BlockNumber, NoteTag); +/// A note to import by details: the stored record it updates when there is one, its details, the +/// block from which to look for its commitment, and the tag to track it under. +pub(crate) type NoteImportByDetailsRequest = + (Option, NoteDetails, BlockNumber, NoteTag); // HELPERS // ================================================================================================ From cb04134aea9a59479aead23ce763ff56ced0df87 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 10 Sep 2026 13:32:26 -0300 Subject: [PATCH 39/43] chore: rename functions and structs --- crates/rust-client/src/note_transport/mod.rs | 68 ++++++++++--------- crates/rust-client/src/sync/mod.rs | 17 ++--- .../miden-client-tests/src/tests/transport.rs | 2 +- 3 files changed, 46 insertions(+), 41 deletions(-) diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index eea3c49434..6e5de44ab7 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -361,9 +361,8 @@ where let cursor = self.store.get_note_transport_cursor().await?; let mut id_by_commitment = BTreeMap::new(); - let (note_files, new_cursor) = self - .fetch_note_transport_updates(cursor, ¬e_tags, &mut id_by_commitment) - .await?; + let (note_files, new_cursor) = + self.fetch_transport_notes(cursor, ¬e_tags, &mut id_by_commitment).await?; self.import_note_records(¬e_files).await?; self.store.update_note_transport_cursor(new_cursor).await?; @@ -410,7 +409,7 @@ where /// Drain a single tag's full history from the transport, paging until the cursor stops /// advancing. Uses a local cursor and never touches the global one, so it cannot regress - /// steady-state progress. Returns the updates from every fetched page, merged in page order and + /// steady-state progress. Returns the note files from every fetched page, in page order and /// none of them written. async fn backfill_tag( &self, @@ -421,7 +420,7 @@ where let mut cursor = NoteTransportCursor::init(); for _ in 0..Self::MAX_BACKFILL_ITERATIONS { let (page_files, new_cursor) = - self.fetch_note_transport_updates(cursor, &[tag], id_by_commitment).await?; + self.fetch_transport_notes(cursor, &[tag], id_by_commitment).await?; note_files.extend(page_files); // Terminate on any lack of forward progress. A well-behaved server returns `new_cursor // == cursor` when there are no new notes for this tag (since `rcursor = max(cursor, @@ -442,18 +441,18 @@ where /// Fetches and returns one batch of notes from the note transport layer for the provided tags /// without applying any update to the store. /// - /// The server paginates; this method issues one RPC and returns the updates together with the - /// new cursor. The returned cursor equals the input cursor when the batch was empty (i.e. no - /// new notes). Callers that want to drain a tag's full backlog should loop until `new_cursor == - /// cursor` (see [`Client::backfill_tag`]). Callers that do steady-state polling (see - /// [`Client::sync_state`] / [`Client::fetch_private_notes`]) should call this once per tick - /// with the stored cursor. + /// The server paginates; this method issues one transport call and returns the note files + /// together with the new cursor. The returned cursor equals the input cursor when the batch was + /// empty (i.e. no new notes). Callers that want to drain a tag's full backlog should loop until + /// `new_cursor == cursor` (see [`Client::backfill_tag`]). Callers that do steady-state polling + /// (see [`Client::sync_state`] / [`Client::fetch_private_notes`]) should call this once per + /// tick with the stored cursor. /// /// Each downloaded note's id is recorded in `id_by_commitment` so the caller can resolve the /// written records back to note ids once the final record set is known. Persistence of the /// returned cursor is left to the caller so that drain loops can guard against regression of an /// already-advanced stored cursor. - async fn fetch_note_transport_updates( + async fn fetch_transport_notes( &self, cursor: NoteTransportCursor, tags: &[NoteTag], @@ -526,12 +525,12 @@ where /// imports the returned files and then persists the cursor and the covered-tag set. /// /// Returns empty data when note transport is not configured. - pub(crate) async fn fetch_note_transport_notes( + pub(crate) async fn fetch_note_transport_updates( &self, - ) -> Result { - let mut fetch = NoteTransportFetch::default(); + ) -> Result { + let mut note_transport_update = NoteTransportLayerUpdate::default(); if !self.is_note_transport_enabled() { - return Ok(fetch); + return Ok(note_transport_update); } // Drain any private notes whose previous relay attempt failed. A flush error is logged, not @@ -547,42 +546,47 @@ where let (mut covered, pruned, new_tags) = self.plan_backfill().await?; let backfilled = !new_tags.is_empty(); for tag in new_tags { - fetch + note_transport_update .note_files - .extend(self.backfill_tag(tag, &mut fetch.id_by_commitment).await?); + .extend(self.backfill_tag(tag, &mut note_transport_update.id_by_commitment).await?); covered.insert(tag); } if pruned || backfilled { - fetch.covered_tags = Some(covered); + note_transport_update.covered_tags = Some(covered); } let cursor = self.store.get_note_transport_cursor().await?; let note_tags: Vec = self.store.get_unique_note_tags().await?.into_iter().collect(); let (note_files, new_cursor) = self - .fetch_note_transport_updates(cursor, ¬e_tags, &mut fetch.id_by_commitment) + .fetch_transport_notes(cursor, ¬e_tags, &mut note_transport_update.id_by_commitment) .await?; - fetch.note_files.extend(note_files); - fetch.cursor = Some(new_cursor); + note_transport_update.note_files.extend(note_files); + note_transport_update.cursor = Some(new_cursor); - Ok(fetch) + Ok(note_transport_update) } - /// Imports what [`Client::fetch_note_transport_notes`] returned, returning the ids of the - /// imported notes and the records written. + /// Writes everything [`Client::fetch_note_transport_updates`] returned, in three steps: + /// + /// 1. Imports the fetched notes, which resolves their on-chain state and stores the records. + /// 2. Saves the covered-tag set, when the backfill changed it. + /// 3. Advances the stored note transport cursor, when a page was fetched. /// /// The notes are written before the covered-tag set and the cursor, so a crash between them /// re-fetches instead of skipping notes that were never written. - pub(crate) async fn import_note_transport_notes( + /// + /// Returns the ids of the imported notes and the records written. + pub(crate) async fn apply_note_transport_update( &mut self, - fetch: NoteTransportFetch, + update: NoteTransportLayerUpdate, ) -> Result<(Vec, Vec), ClientError> { - let NoteTransportFetch { + let NoteTransportLayerUpdate { note_files, id_by_commitment, covered_tags, cursor, - } = fetch; + } = update; let written = self.import_note_records(¬e_files).await?; let mut imported_ids: Vec = written @@ -636,10 +640,10 @@ where /// What the note transport fetch returned, before anything is written. /// -/// Built by [`Client::fetch_note_transport_notes`] and consumed by -/// [`Client::import_note_transport_notes`]. +/// Built by [`Client::fetch_note_transport_updates`] and consumed by +/// [`Client::apply_note_transport_update`]. #[derive(Default)] -pub(crate) struct NoteTransportFetch { +pub(crate) struct NoteTransportLayerUpdate { /// Notes to import, backfill pages first and then the steady-state page. note_files: Vec, /// Note ids by details commitment, taken from the note headers the transport returned. Used to diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 7c3d74635b..79c96add95 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -232,9 +232,8 @@ where } self.ensure_genesis_in_place().await?; - let fetch = self.fetch_note_transport_notes().await?; - - let (imported_ids, _) = self.import_note_transport_notes(fetch).await?; + let note_transport_update = self.fetch_note_transport_updates().await?; + let (imported_ids, _) = self.apply_note_transport_update(note_transport_update).await?; Ok(imported_ids) } @@ -267,14 +266,16 @@ where self.ensure_rpc_limits_in_place().await?; let state_sync = self.state_sync(); - let (transport_fetch, chain_sync_data) = futures::join!( - self.fetch_note_transport_notes(), + let (note_transport_update, chain_sync_data) = futures::join!( + self.fetch_note_transport_updates(), self.fetch_chain_updates(&state_sync), ); // An NTL failure does not end the sync - let (new_private_notes, imported) = match transport_fetch { - Ok(fetch) => self.import_note_transport_notes(fetch).await?, + let (new_private_notes, imported_notes) = match note_transport_update { + Ok(note_transport_update) => { + self.apply_note_transport_update(note_transport_update).await? + }, Err(err) => { warn!(?err, "note transport fetch failed; syncing the chain without it"); (Vec::new(), Vec::new()) @@ -286,7 +287,7 @@ where // The chain sync built its note updates from a store snapshot taken before the import, so // the imported records are added here. Without them this sync has no record to apply its // verdicts to, and a note committed within this sync's own block range stays expected. - chain_sync_data.note_updates.track_existing_input_notes(imported); + chain_sync_data.note_updates.track_existing_input_notes(imported_notes); state_sync.derive_state_updates(&mut chain_sync_data).await?; state_sync.fetch_nullifiers(&mut chain_sync_data).await?; diff --git a/crates/testing/miden-client-tests/src/tests/transport.rs b/crates/testing/miden-client-tests/src/tests/transport.rs index 6f659e07d1..7d13e0f1db 100644 --- a/crates/testing/miden-client-tests/src/tests/transport.rs +++ b/crates/testing/miden-client-tests/src/tests/transport.rs @@ -580,7 +580,7 @@ async fn fetch_private_notes_finds_note_committed_at_sync_height() { let details_bytes = details.to_bytes(); mock_transport_node.write().add_note(*private_note.header(), details_bytes); - // 6. Second sync_state: fetch_transport_notes imports the note, then chain sync runs. The + // 6. Second sync_state: the transport page is imported, then chain sync runs. The // chain scan starts from a lookback window rather than from the sync height, so it still sees // the note at block 1. let summary = client.sync_state().await.unwrap(); From 6edabf98a6208168b9abad4654173f292c5b2866 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 10 Sep 2026 13:33:12 -0300 Subject: [PATCH 40/43] chore: update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 639366a281..546c7c0ce0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Breaking Changes * [BREAKING][type][rust] Added the `TransactionRequestError::SwapNoteWithZeroAsset` variant, so exhaustive matches on `TransactionRequestError` must handle it ([#2459](https://github.com/0xMiden/rust-sdk/pull/2459)). - +* [BREAKING][behavior][rust] A Note Transport Layer failure no longer fails `Client::sync_state`. The error is logged and the chain sync still applies; the transport cursor is left where it was, so the next sync requests the same page again ([#2453](https://github.com/0xMiden/rust-sdk/pull/2453)). ### Fixes @@ -16,7 +16,7 @@ ### Enhancements -* [rust] The chain sync and the note transport sync are each split into a fetch phase and a store phase, so every NTL and RPC call happens before the first store write. `Client::sync_state` runs the two fetch phases concurrently and applies both sets of updates afterwards, instead of running a full note transport sync before the chain sync ([#2453](https://github.com/0xMiden/rust-sdk/pull/2453)). +* [rust] `Client::sync_state` fetches a Note Transport Layer page and the node's chain update concurrently, instead of running a full note transport sync before the chain sync. The transport notes are imported first and their records join the chain sync's note updates, so a note delivered and committed within the same sync is reported by that sync ([#2453](https://github.com/0xMiden/rust-sdk/pull/2453)). ## 0.16.0 (2026-09-07) From d2f2b91ede593b1db42f6f3b367ab4da4ea53462 Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 10 Sep 2026 13:59:20 -0300 Subject: [PATCH 41/43] refactor(rust-client): drop import_note_records and read the imported records back in sync_state --- crates/rust-client/src/note/import.rs | 25 ++++++-------------- crates/rust-client/src/note_transport/mod.rs | 10 ++++---- crates/rust-client/src/sync/mod.rs | 2 ++ 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 8156bb2950..1b1b827c2b 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -68,18 +68,6 @@ where &mut self, note_files: &[NoteFile], ) -> Result, ClientError> { - let records = self.import_note_records(note_files).await?; - Ok(records.iter().map(InputNoteRecord::details_commitment).collect()) - } - - /// Imports `note_files` and returns the records written to the store. - /// - /// A caller that must act on the imported notes in the same operation needs the records, not - /// just their commitments: a sync uses them to extend its own note updates. - pub(crate) async fn import_note_records( - &mut self, - note_files: &[NoteFile], - ) -> Result, ClientError> { self.ensure_genesis_in_place().await?; // Deduplicate the incoming files, keeping note IDs and details commitments in separate @@ -168,19 +156,20 @@ where imported_notes.extend(notes_by_proof); } - for note in &imported_notes { - // A record still expected needs its tag tracked so a later sync finds it. A committed - // one is no longer in that state, so it is skipped here. + let mut imported_commitments = Vec::with_capacity(imported_notes.len()); + for note in imported_notes { + let details_commitment = note.details_commitment(); if let InputNoteState::Expected(ExpectedNoteState { tag: Some(tag), .. }) = note.state() { self.store - .add_note_tag(NoteTagRecord::with_note_source(*tag, note.details_commitment())) + .add_note_tag(NoteTagRecord::with_note_source(*tag, details_commitment)) .await?; } + self.store.upsert_input_notes(&[note]).await?; + imported_commitments.push(details_commitment); } - self.store.upsert_input_notes(&imported_notes).await?; - Ok(imported_notes) + Ok(imported_commitments) } // HELPERS diff --git a/crates/rust-client/src/note_transport/mod.rs b/crates/rust-client/src/note_transport/mod.rs index 6e5de44ab7..c1f6c7fd1c 100644 --- a/crates/rust-client/src/note_transport/mod.rs +++ b/crates/rust-client/src/note_transport/mod.rs @@ -364,7 +364,7 @@ where let (note_files, new_cursor) = self.fetch_transport_notes(cursor, ¬e_tags, &mut id_by_commitment).await?; - self.import_note_records(¬e_files).await?; + self.import_notes(¬e_files).await?; self.store.update_note_transport_cursor(new_cursor).await?; Ok(()) @@ -576,11 +576,11 @@ where /// The notes are written before the covered-tag set and the cursor, so a crash between them /// re-fetches instead of skipping notes that were never written. /// - /// Returns the ids of the imported notes and the records written. + /// Returns the ids of the imported notes and the details commitments of the records written. pub(crate) async fn apply_note_transport_update( &mut self, update: NoteTransportLayerUpdate, - ) -> Result<(Vec, Vec), ClientError> { + ) -> Result<(Vec, Vec), ClientError> { let NoteTransportLayerUpdate { note_files, id_by_commitment, @@ -588,10 +588,10 @@ where cursor, } = update; - let written = self.import_note_records(¬e_files).await?; + let written = self.import_notes(¬e_files).await?; let mut imported_ids: Vec = written .iter() - .filter_map(|note| id_by_commitment.get(¬e.details_commitment()).copied()) + .filter_map(|commitment| id_by_commitment.get(commitment).copied()) .collect(); if let Some(covered_tags) = covered_tags { diff --git a/crates/rust-client/src/sync/mod.rs b/crates/rust-client/src/sync/mod.rs index 79c96add95..1911ce8c64 100644 --- a/crates/rust-client/src/sync/mod.rs +++ b/crates/rust-client/src/sync/mod.rs @@ -287,6 +287,8 @@ where // The chain sync built its note updates from a store snapshot taken before the import, so // the imported records are added here. Without them this sync has no record to apply its // verdicts to, and a note committed within this sync's own block range stays expected. + let imported_notes = + self.get_input_notes(NoteFilter::DetailsCommitments(imported_notes)).await?; chain_sync_data.note_updates.track_existing_input_notes(imported_notes); state_sync.derive_state_updates(&mut chain_sync_data).await?; From f41fff857408a52f4fd0595bd0abf3c968c1264e Mon Sep 17 00:00:00 2001 From: ricomateo Date: Thu, 10 Sep 2026 14:52:33 -0300 Subject: [PATCH 42/43] chore: simplify sync_expected_notes and import_note_records_by_details --- crates/rust-client/src/note/import.rs | 107 +++++++++----------- crates/rust-client/src/sync/block_header.rs | 4 +- 2 files changed, 52 insertions(+), 59 deletions(-) diff --git a/crates/rust-client/src/note/import.rs b/crates/rust-client/src/note/import.rs index 1b1b827c2b..f473c7f16c 100644 --- a/crates/rust-client/src/note/import.rs +++ b/crates/rust-client/src/note/import.rs @@ -25,7 +25,7 @@ use miden_protocol::note::{ use miden_standards::note::NoteFile; use miden_tx::auth::TransactionAuthenticator; -use crate::rpc::domain::note::{FetchedNote, ResolvedSyncNotesBlock, SyncedNote}; +use crate::rpc::domain::note::{FetchedNote, ResolvedSyncNotesBlock}; use crate::rpc::{NoteContentFetch, RpcError}; use crate::store::input_note_states::ExpectedNoteState; use crate::store::{InputNoteRecord, InputNoteState, NoteFilter}; @@ -345,46 +345,18 @@ where requested_notes: Vec, ) -> Result, ClientError> { let mut lowest_request_block: BlockNumber = u32::MAX.into(); - let mut sync_tags = BTreeSet::new(); - let mut requested_commitments = Vec::with_capacity(requested_notes.len()); + let mut note_requests = vec![]; for (_, details, after_block_num, tag) in &requested_notes { - sync_tags.insert(*tag); - requested_commitments.push(details.commitment()); + note_requests.push((details.commitment(), *tag)); lowest_request_block = lowest_request_block.min(*after_block_num); } - let blocks = self.sync_expected_notes(lowest_request_block, &sync_tags).await?; - - // An expected note has no metadata and thus no `NoteId`, so each returned note is matched - // to its request by rebuilding the id from the committed metadata. Only the blocks holding - // a match are kept: the rest hold notes under the same tag that answer no request. - let mut committed_notes_data = BTreeMap::new(); - let mut matched_blocks = Vec::new(); - for block in blocks { - let mut block_matched = false; - for (note_id, sync_note) in &block.notes { - let metadata = sync_note.committed.metadata(); - let Some(commitment) = requested_commitments - .iter() - .find(|commitment| NoteId::new(**commitment, metadata) == *note_id) - else { - continue; - }; - - committed_notes_data - .insert(*commitment, (sync_note.clone(), block.block_header.clone())); - block_matched = true; - } - - if block_matched { - matched_blocks.push(block); - } - } + let blocks = self.sync_expected_notes(lowest_request_block, ¬e_requests).await?; // The blocks arrive with the notes, so a committed note needs no further block lookup. They // are stored first, so a record is never persisted as committed before the header that // proves its inclusion is tracked and stored. let mut partial_mmr = self.get_current_partial_mmr().await?; - self.insert_note_blocks(matched_blocks, &mut partial_mmr).await?; + self.insert_note_blocks(&blocks, &mut partial_mmr).await?; self.cache_partial_mmr(partial_mmr).await?; let mut note_records = vec![]; @@ -404,19 +376,22 @@ where }); // Notes the node has not reported as committed keep their expected record untouched. - let Some(( - SyncedNote { - committed: committed_note, attachments, .. - }, - block_header, - )) = committed_notes_data.remove(¬e_record.details_commitment()) - else { + let commitment = note_record.details_commitment(); + let Some((sync_note, block_header)) = blocks.iter().find_map(|block| { + let sync_note = block.notes.values().find(|sync_note| { + NoteId::new(commitment, sync_note.committed.metadata()) + == *sync_note.committed.note_id() + })?; + Some((sync_note, &block.block_header)) + }) else { note_records.push(note_record); continue; }; + let committed_note = &sync_note.committed; // A note that carries no attachments has nothing to apply to the record. - let attachments = (!attachments.is_empty()).then_some(attachments); + let attachments = + (!sync_note.attachments.is_empty()).then(|| sync_note.attachments.clone()); let metadata = *committed_note.metadata(); let mut note_changed = note_record @@ -427,7 +402,7 @@ where } // `block_header_received` transitions the record's state, so it must always run. - note_changed |= note_record.block_header_received(&block_header)?; + note_changed |= note_record.block_header_received(block_header)?; // Once committed, the note no longer needs its expected-note tag. if note_changed { @@ -504,8 +479,10 @@ where async fn sync_expected_notes( &self, request_block_num: BlockNumber, - sync_tags: &BTreeSet, + // Expected notes' details commitments with their tags. + expected_notes: &[(NoteDetailsCommitment, NoteTag)], ) -> Result, ClientError> { + let sync_tags: BTreeSet = expected_notes.iter().map(|(_, tag)| *tag).collect(); let current_block_num = self.get_sync_height().await?; // Notes expected only after a block we have not reached can't be committed within our @@ -514,34 +491,50 @@ where return Ok(Vec::new()); } - let mut blocks = self + let blocks = self .rpc_api .sync_notes_with_content( request_block_num, current_block_num, - sync_tags, + &sync_tags, NoteContentFetch::AttachmentsOnly, ) .await .map_err(ClientError::RpcError)?; - blocks.retain_mut(|block| { + let mut matched_blocks = vec![]; + for block in blocks { + let mut block_matches = false; if block.block_header.block_num() > current_block_num { - return false; + break; } - // A note carries its own commit height in its inclusion proof, which is a separate - // field from the block header checked above. Authenticating the note later looks that - // height up in the partial MMR, so a height beyond our synced view has to be dropped - // here rather than trusted. - block - .notes - .retain(|_, sync_note| sync_note.committed.block_num() <= current_block_num); + for sync_note in block.notes.values() { + let committed = &sync_note.committed; + + // The note carries its own commit height in its inclusion proof, which is a + // separate field from the block header checked above. Authenticating the note later + // looks that height up in the partial MMR, so a height beyond our synced view has + // to be dropped here rather than trusted. + if committed.block_num() > current_block_num { + continue; + } + + let Some((..)) = expected_notes.iter().find(|(commitment, _)| { + NoteId::new(*commitment, committed.metadata()) == *committed.note_id() + }) else { + continue; + }; + + block_matches = true; + } - !block.notes.is_empty() - }); + if block_matches { + matched_blocks.push(block); + } + } - Ok(blocks) + Ok(matched_blocks) } } diff --git a/crates/rust-client/src/sync/block_header.rs b/crates/rust-client/src/sync/block_header.rs index 58cb97058f..bdedcb1755 100644 --- a/crates/rust-client/src/sync/block_header.rs +++ b/crates/rust-client/src/sync/block_header.rs @@ -133,7 +133,7 @@ impl Client { /// authentication nodes that tracking produced. pub(crate) async fn insert_note_blocks( &mut self, - blocks: Vec, + blocks: &[ResolvedSyncNotesBlock], partial_mmr: &mut PartialMmr, ) -> Result<(), ClientError> { let mut authenticated_blocks = Vec::with_capacity(blocks.len()); @@ -151,7 +151,7 @@ impl Client { block.block_header.commitment(), &block.mmr_path, )?; - authenticated_blocks.push((block.block_header, path_nodes)); + authenticated_blocks.push((block.block_header.clone(), path_nodes)); } for (block_header, path_nodes) in authenticated_blocks { From 2cc461ba9637d86f3e97468efd65f6c531ec65f6 Mon Sep 17 00:00:00 2001 From: Ignacio Amigo Date: Fri, 11 Sep 2026 15:53:58 -0300 Subject: [PATCH 43/43] fix(rust-client): refresh tracked notes after transport imports --- CHANGELOG.md | 1 + .../src/note/note_update_tracker.rs | 12 +-- .../miden-client-tests/src/tests/transport.rs | 93 +++++++++++++++++++ 3 files changed, 98 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4272a4d18..bfc170d784 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Fixes +* [FIX][rust] Refreshed tracked input notes after transport imports so the same sync detects their consumption ([#2453](https://github.com/0xMiden/rust-sdk/pull/2453)). * [FIX][rust] A private note fetched from the Note Transport Layer whose nullifier is already on chain is now imported as consumed instead of committed, so `get_consumable_notes` no longer reports notes the node will reject ([#2453](https://github.com/0xMiden/rust-sdk/pull/2453)). * [FIX][rust] Added validation of cached transaction encryption keys during deserialization. Unsupported encryption schemes and empty or oversized key IDs are rejected before reading the key ID bytes ([#2411](https://github.com/0xMiden/rust-sdk/pull/2411)). * [FIX][cli] `miden-client import` now rejects invocations without a file path instead of silently succeeding ([#2450](https://github.com/0xMiden/rust-sdk/pull/2450)). diff --git a/crates/rust-client/src/note/note_update_tracker.rs b/crates/rust-client/src/note/note_update_tracker.rs index ec9e8559a4..71dd65e549 100644 --- a/crates/rust-client/src/note/note_update_tracker.rs +++ b/crates/rust-client/src/note/note_update_tracker.rs @@ -377,20 +377,16 @@ impl NoteUpdateTracker { }) } - /// Tracks additional already-persisted input notes. + /// Refreshes the tracker with persisted input notes. /// - /// Used to extend a sync's nullifier check to notes that are about to be written by another - /// path (e.g. the note transport sync) and are therefore absent from the store snapshot this - /// tracker was built from. Notes already tracked for the same details commitment are skipped, - /// so a record built by this sync is never replaced by a stale one. + /// Call this method before deriving state updates. Imported records can replace older records + /// from the initial store snapshot. The records are already persisted, so they need no store + /// update until their state changes. pub(crate) fn track_existing_input_notes( &mut self, notes: impl IntoIterator, ) { for note in notes { - if self.input_notes.contains_key(¬e.details_commitment()) { - continue; - } self.insert_input_note(note, NoteUpdateType::None); } } diff --git a/crates/testing/miden-client-tests/src/tests/transport.rs b/crates/testing/miden-client-tests/src/tests/transport.rs index fd203e60a2..79b66cefcf 100644 --- a/crates/testing/miden-client-tests/src/tests/transport.rs +++ b/crates/testing/miden-client-tests/src/tests/transport.rs @@ -10,6 +10,8 @@ use miden_client::note::{ Note, NoteDetails, NoteExecutionHint, + NoteFile, + NoteSyncHint, NoteTag, NoteType, }; @@ -787,6 +789,97 @@ async fn ntl_note_already_spent_below_the_checkpoint_is_not_left_committed() { ); } +/// A transport import can resolve an expected note below the checkpoint while the chain sync +/// reports its consumption above the checkpoint. +#[tokio::test] +async fn ntl_refresh_of_expected_note_detects_consumption_in_same_sync() { + let sender_id: AccountId = ACCOUNT_ID_SENDER.try_into().unwrap(); + let faucet_id: AccountId = ACCOUNT_ID_PRIVATE_FUNGIBLE_FAUCET.try_into().unwrap(); + + let mut builder = MockChainBuilder::new(); + let account = builder.add_existing_mock_account(Auth::IncrNonce).unwrap(); + let asset = Asset::Fungible(FungibleAsset::new(faucet_id, 100u64).unwrap()); + let note = builder + .add_p2id_note(sender_id, account.id(), &[asset], ProtocolNoteType::Private) + .unwrap(); + + let mut mock_chain = builder.build().unwrap(); + mock_chain.prove_next_block().unwrap(); + + let consume_tx = Box::pin( + mock_chain + .build_transaction(MockTransactionInput::Account(account)) + .unauthenticated_input_note(note.clone()) + .build() + .unwrap() + .execute(), + ) + .await + .unwrap(); + let mock_transport_node = Arc::new(RwLock::new(MockNoteTransportNode::new())); + let rpc_api = Arc::new(MockRpcApi::new(mock_chain)); + let transport_client = MockNoteTransportApi::new(mock_transport_node.clone()); + + let rng = RandomCoin::new([1, 2, 3, 4].map(Felt::new_unchecked).into()); + let keystore = FilesystemKeyStore::new(temp_dir()).unwrap(); + + let builder: ClientBuilder = ClientBuilder::new() + .rpc(rpc_api.clone()) + .rng(Box::new(rng)) + .sqlite_store(create_test_store_path()) + .authenticator(Arc::new(keystore)) + .tx_discard_delta(None) + .note_transport(Arc::new(transport_client)); + + let mut client = builder.build().await.unwrap(); + client.ensure_genesis_in_place().await.unwrap(); + seed_mock_transaction_encryption_key(&mut client).await; + client.add_note_tag(note.metadata().tag()).await.unwrap(); + + client.sync_state().await.unwrap(); + let checkpoint = client.get_sync_height().await.unwrap(); + assert_eq!(checkpoint, BlockNumber::from(1)); + // A later search floor keeps the initial import expected. The transport supplies an earlier + // floor that resolves its commitment. + client + .import_notes(&[NoteFile::ExpectedNote { + details: NoteDetails::from(note.clone()), + sync_hint: NoteSyncHint::new(checkpoint + 1, note.metadata().tag()), + }]) + .await + .unwrap(); + let expected = client.get_input_notes(NoteFilter::Expected).await.unwrap(); + assert_eq!(expected.len(), 1); + assert_eq!(expected[0].details_commitment(), note.details_commitment()); + assert!(expected[0].metadata().is_none()); + + // The spend is above the checkpoint. The chain sync must check the nullifier supplied by the + // transport import. + rpc_api + .mock_chain + .write() + .add_pending_executed_transaction(&consume_tx) + .unwrap(); + rpc_api.prove_block(); + + let details_bytes = NoteDetails::from(note.clone()).to_bytes(); + mock_transport_node.write().add_note_after( + *note.header(), + details_bytes, + Some(BlockNumber::GENESIS), + ); + + client.sync_state().await.unwrap(); + + let record = client.get_input_note(note.id()).await.unwrap().unwrap(); + assert!(record.is_consumed(), "the refreshed note must be consumed in the same sync"); + assert_eq!(client.get_sync_height().await.unwrap(), BlockNumber::from(2)); + + client.sync_state().await.unwrap(); + let record = client.get_input_note(note.id()).await.unwrap().unwrap(); + assert!(record.is_consumed()); +} + /// A private note must reach the recipient even when the sender's first relay attempt fails, /// provided the transport later recovers. ///