From 8c2668b43f698f8641b570eebd860e8ae8e3055c Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Tue, 23 Sep 2025 11:20:52 +0400 Subject: [PATCH] fix: fix fee claiming, indexer syncs VN fee pool substates --- .../src/network_state_sync/block_scanner.rs | 91 +++---------------- .../src/network_state_sync/worker.rs | 21 ++++- .../2023-02-16-145719_initial/up.sql | 2 - .../src/storage_sqlite/models/substate.rs | 2 - .../tari_indexer/src/storage_sqlite/reader.rs | 2 +- .../tari_indexer/src/storage_sqlite/schema.rs | 1 - .../src/storage_sqlite/store_factory.rs | 6 +- .../tari_indexer/src/storage_sqlite/writer.rs | 27 +++++- applications/tari_indexer/web_ui/package.json | 2 +- .../wallet_daemon_create_key.rs | 27 +++--- .../src/json_rpc/handlers.rs | 2 + .../src/p2p/rpc/block_sync_task.rs | 1 + .../tari_validator_node/web_ui/package.json | 2 +- .../web_ui/src/routes/VN/Components/Info.tsx | 61 ++----------- .../tari_walletd/src/handlers/validator.rs | 60 ++++++------ applications/tari_walletd/web_ui/package.json | 2 +- .../tari_walletd/web_ui/src/utils/helpers.tsx | 1 + .../VNGetIdentityResponse.ts | 1 + clients/validator_node_client/src/types.rs | 1 + crates/engine_types/src/substate.rs | 6 +- crates/state_store_rocksdb/src/options.rs | 3 +- crates/storage/src/consensus_models/block.rs | 7 +- .../storage/src/consensus_models/command.rs | 1 - crates/wallet/sdk/src/apis/accounts.rs | 3 +- crates/wallet/sdk/src/apis/substate.rs | 14 ++- crates/wallet/storage_sqlite/src/writer.rs | 3 + 26 files changed, 156 insertions(+), 193 deletions(-) diff --git a/applications/tari_indexer/src/network_state_sync/block_scanner.rs b/applications/tari_indexer/src/network_state_sync/block_scanner.rs index c74fddf0d5..1950e71bbe 100644 --- a/applications/tari_indexer/src/network_state_sync/block_scanner.rs +++ b/applications/tari_indexer/src/network_state_sync/block_scanner.rs @@ -4,21 +4,16 @@ use futures::StreamExt; use log::*; use tari_consensus_types::BlockId; -use tari_engine_types::substate::SubstateValue; use tari_epoch_manager::{service::EpochManagerHandle, EpochManagerReader}; use tari_ootle_common_types::{committee::Committee, Epoch, PeerAddress, ShardGroup}; use tari_ootle_p2p::{proto, proto::rpc::SyncBlocksRequest}; -use tari_ootle_storage::{ - consensus_models::{Block, SubstateUpdateProof}, - time::{OffsetDateTime, PrimitiveDateTime}, -}; -use tari_template_lib::types::TemplateAddress; +use tari_ootle_storage::consensus_models::{Block, SubstateUpdateProof}; use tari_validator_node_rpc::client::{TariValidatorNodeRpcClientFactory, ValidatorNodeClientFactory}; use crate::{ block_data::BlockData, storage_sqlite::{ - models::{NewScannedBlockId, NewSubstate}, + models::NewScannedBlockId, IndexerStore, IndexerStoreReadTransaction, IndexerStoreWriteTransaction, @@ -107,12 +102,19 @@ impl BlockScanner { count += new_blocks.len(); for block_data in new_blocks { - let timestamp = unix_epoch_to_primitive_date_time(block_data.block.timestamp()); // TODO: store blocks // TODO: remove substates (I think). These can be requested lazily and cached (LRU) as needed to allow // TODO: an committed transaction should queue a shard state sync in the affected shards // an upper bound on substates stored in the indexer. - self.store_substates_in_db(&block_data.diff, timestamp)?; + info!( + target: LOG_TARGET, + "Storing {} substate update(s) for block {} (epoch={}, height={})", + block_data.diff.len(), + block_data.block.id(), + block_data.block.epoch(), + block_data.block.height() + ); + self.store_substates_in_db(&block_data.diff)?; } } @@ -125,42 +127,24 @@ impl BlockScanner { .map_err(|e| e.into()) } - fn store_substates_in_db( - &self, - updates: &[SubstateUpdateProof], - timestamp: PrimitiveDateTime, - ) -> Result<(), anyhow::Error> { + fn store_substates_in_db(&self, updates: &[SubstateUpdateProof]) -> Result<(), anyhow::Error> { let mut tx = self.substate_store.create_write_tx()?; // store/update up substates if any for update in updates { match update { SubstateUpdateProof::Create(create) => { - let maybe_substate_value = create.substate.value.value(); - if maybe_substate_value.is_none() { + if create.substate.value.value().is_none() { warn!( target: LOG_TARGET, "⚠️ Received UP substate {} without value. This indicates that the substate has been pruned. Some event data is not available.", create.substate.as_versioned_substate_id_ref(), ); } - let template_address = maybe_substate_value.and_then(Self::extract_template_address_from_substate); - let module_name = maybe_substate_value.and_then(Self::extract_module_name_from_substate); - let substate_row = NewSubstate { - address: create.substate.substate_id.to_string(), - version: create.substate.version as i32, - data: maybe_substate_value - .map(Self::encode_substate) - .transpose()? - .unwrap_or_default(), - template_address: template_address.map(|s| s.to_string()), - module_name, - timestamp, - }; debug!( target: LOG_TARGET, "Saving substate: {:?}", - substate_row + create.substate ); - tx.upsert_substate(substate_row)?; + tx.upsert_substate(&create.substate)?; }, SubstateUpdateProof::Destroy(_) => {}, } @@ -169,25 +153,6 @@ impl BlockScanner { Ok(()) } - fn extract_template_address_from_substate(substate: &SubstateValue) -> Option { - match substate { - SubstateValue::Component(c) => Some(c.template_address), - _ => None, - } - } - - fn extract_module_name_from_substate(substate: &SubstateValue) -> Option { - match substate { - SubstateValue::Component(c) => Some(c.module_name.to_owned()), - _ => None, - } - } - - fn encode_substate(substate: &SubstateValue) -> Result { - let pretty_json = serde_json::to_string_pretty(&substate)?; - Ok(pretty_json) - } - async fn get_oldest_scanned_epoch(&self) -> Result, anyhow::Error> { self.substate_store .with_read_tx(|tx| tx.get_oldest_scanned_epoch()) @@ -361,29 +326,3 @@ impl BlockScanner { Ok(blocks) } } - -fn unix_epoch_to_primitive_date_time(timestamp: u64) -> PrimitiveDateTime { - let timestamp = i64::try_from(timestamp).unwrap_or_else(|e| { - // TODO: this is very possible because we trust that the timestamp is roughly correct, however - // it is purely informational and not enforced in consensus therefore could be any value and - // therefore cannot be relied for ordering (use (epoch,height) instead). - warn!( - target: LOG_TARGET, - "Failed to convert block timestamp to PrimitiveDateTime: {}", - e - ); - i64::MAX // = August 17, 292278994, 07:12:55.807 UTC - }); - OffsetDateTime::from_unix_timestamp(timestamp) - .map(|osdt| PrimitiveDateTime::new(osdt.date(), osdt.time())) - .unwrap_or_else(|e| { - warn!( - target: LOG_TARGET, - "Failed to convert block timestamp to OffsetDateTime: {}. Using UNIX_EPOCH", - e - ); - // An error cannot be because the timestamp is too small, because we use an u64 and a zero unix - // timestamp represents a greater date (1970 AD) than the minimum (9999 BC) - PrimitiveDateTime::MAX - }) -} diff --git a/applications/tari_indexer/src/network_state_sync/worker.rs b/applications/tari_indexer/src/network_state_sync/worker.rs index ca532dc1e2..3223348d74 100644 --- a/applications/tari_indexer/src/network_state_sync/worker.rs +++ b/applications/tari_indexer/src/network_state_sync/worker.rs @@ -22,7 +22,7 @@ use tari_ootle_common_types::{ }; use tari_ootle_p2p::{proto::rpc, TariMessagingSpec}; use tari_ootle_storage::{ - consensus_models::{EpochCheckpoint, SubstateUpdateProof, SubstateValueFilterFlags}, + consensus_models::{EpochCheckpoint, SubstateData, SubstateUpdateProof, SubstateValueFilterFlags}, StorageError, }; use tari_rpc_framework::__macro_reexports::future::Either; @@ -282,6 +282,7 @@ impl NetworkWideStateSync { let mut update_buf = Vec::new(); let mut utxos_buf = Vec::new(); let mut transactions_buf = Vec::new(); + let mut validator_fee_pools_buf = Vec::new(); let mut has_synced_global_shard = false; @@ -297,6 +298,7 @@ impl NetworkWideStateSync { &mut update_buf, &mut utxos_buf, &mut transactions_buf, + &mut validator_fee_pools_buf, shard_group, &mut session, ) @@ -311,6 +313,7 @@ impl NetworkWideStateSync { &mut update_buf, &mut utxos_buf, &mut transactions_buf, + &mut validator_fee_pools_buf, shard_group, &mut session, ) @@ -328,6 +331,7 @@ impl NetworkWideStateSync { update_buf: &mut Vec<(Epoch, SubstateUpdateProof)>, utxos_buf: &mut Vec, transactions_buf: &mut Vec, + validator_fee_pools_buf: &mut Vec, shard_group: ShardGroup, session: &mut ValidatorRpcSession, ) -> Result<(), NetworkStateSyncError> { @@ -348,6 +352,7 @@ impl NetworkWideStateSync { until_epoch: None, value_filters: (SubstateValueFilterFlags::UTXO | SubstateValueFilterFlags::TEMPLATE | + SubstateValueFilterFlags::VALIDATOR_FEE_POOL | SubstateValueFilterFlags::TRANSACTION_RECEIPT) .bits(), }) @@ -389,6 +394,7 @@ impl NetworkWideStateSync { &mut templates_buf, utxos_buf, transactions_buf, + validator_fee_pools_buf, )?; } if msg.has_more { @@ -405,6 +411,11 @@ impl NetworkWideStateSync { tx.batch_insert_substate_transitions(shard, state_version, update_buf.drain(..))?; debug!(target: LOG_TARGET, "✅ Committing {} UTXOs for shard {shard} (epoch: {msg_epoch})", utxos_buf.len()); tx.batch_insert_utxo_updates(utxos_buf.drain(..))?; + // TODO: there are many ways to do this. This is probably not the best way. But this allows wallet to query for validator fee pool values since + // block sync does not sync validator fee pools (due to block diffs being removed on block commit). + for substate_data in validator_fee_pools_buf.drain(..) { + tx.upsert_substate(&substate_data)?; + } // TODO: transaction events and templates debug!(target: LOG_TARGET, "✅ Committing {} transactions for shard {shard} (epoch: {msg_epoch})", transactions_buf.len()); self.stats.increase_events(transactions_buf.len()); @@ -453,6 +464,7 @@ fn extend_bufs_from_substate_update( templates_buf: &mut Vec, utxos_buf: &mut Vec, transactions_buf: &mut Vec, + validator_fee_pools_buf: &mut Vec, ) -> Result<(), NetworkStateSyncError> { match &update { SubstateUpdateProof::Create(create) => match create.substate.value().value() { @@ -489,6 +501,13 @@ fn extend_bufs_from_substate_update( warn!(target: LOG_TARGET, "⚠️ NEVER HAPPEN: Received template substate with invalid address: {}", create.substate.substate_id()); } }, + Some(SubstateValue::ValidatorFeePool(_)) => { + validator_fee_pools_buf.push(SubstateData { + substate_id: create.substate.substate_id().clone(), + version: create.substate.version, + value: create.substate.value().clone(), + }); + }, Some(_) => {}, None => { let id = create.substate.substate_id(); diff --git a/applications/tari_indexer/src/storage_sqlite/migrations/2023-02-16-145719_initial/up.sql b/applications/tari_indexer/src/storage_sqlite/migrations/2023-02-16-145719_initial/up.sql index f9a3626667..07a821ef26 100644 --- a/applications/tari_indexer/src/storage_sqlite/migrations/2023-02-16-145719_initial/up.sql +++ b/applications/tari_indexer/src/storage_sqlite/migrations/2023-02-16-145719_initial/up.sql @@ -6,8 +6,6 @@ create table substates data text not NULL, template_address text NULL, module_name text NULL, - -- Block timestamp - timestamp timestamp not NULL, updated_at timestamp not null default current_timestamp, created_at timestamp not null default current_timestamp ); diff --git a/applications/tari_indexer/src/storage_sqlite/models/substate.rs b/applications/tari_indexer/src/storage_sqlite/models/substate.rs index 8e193cdc0d..0554e0d6a4 100644 --- a/applications/tari_indexer/src/storage_sqlite/models/substate.rs +++ b/applications/tari_indexer/src/storage_sqlite/models/substate.rs @@ -43,7 +43,6 @@ pub struct SubstateRecord { pub data: String, pub template_address: Option, pub module_name: Option, - pub timestamp: PrimitiveDateTime, pub updated_at: PrimitiveDateTime, pub created_at: PrimitiveDateTime, } @@ -72,5 +71,4 @@ pub struct NewSubstate { pub data: String, pub template_address: Option, pub module_name: Option, - pub timestamp: PrimitiveDateTime, } diff --git a/applications/tari_indexer/src/storage_sqlite/reader.rs b/applications/tari_indexer/src/storage_sqlite/reader.rs index 916a0471f8..62049bc190 100644 --- a/applications/tari_indexer/src/storage_sqlite/reader.rs +++ b/applications/tari_indexer/src/storage_sqlite/reader.rs @@ -100,7 +100,7 @@ impl IndexerStoreReadTransaction for SqliteStoreReadTransaction<'_> { let substate_id = SubstateId::from_str(&s.address)?; let version = s.version as u32; let template_address = s.template_address.map(|h| deserialize_hex_try_from(&h)).transpose()?; - let timestamp = s.timestamp; + let timestamp = s.updated_at; Ok(ListSubstateItem { substate_id, module_name: s.module_name, diff --git a/applications/tari_indexer/src/storage_sqlite/schema.rs b/applications/tari_indexer/src/storage_sqlite/schema.rs index 65175c9c69..20972737ef 100644 --- a/applications/tari_indexer/src/storage_sqlite/schema.rs +++ b/applications/tari_indexer/src/storage_sqlite/schema.rs @@ -74,7 +74,6 @@ diesel::table! { data -> Text, template_address -> Nullable, module_name -> Nullable, - timestamp -> Timestamp, updated_at -> Timestamp, created_at -> Timestamp, } diff --git a/applications/tari_indexer/src/storage_sqlite/store_factory.rs b/applications/tari_indexer/src/storage_sqlite/store_factory.rs index d2b478deba..a43ddc4ea2 100644 --- a/applications/tari_indexer/src/storage_sqlite/store_factory.rs +++ b/applications/tari_indexer/src/storage_sqlite/store_factory.rs @@ -17,7 +17,7 @@ use tari_engine_types::{events::Event, substate::SubstateId, Utxo}; use tari_indexer_client::types::{ListSubstateItem, NonFungibleSubstate, TransactionEntry}; use tari_ootle_common_types::{shard::Shard, substate_type::SubstateType, Epoch, ShardGroup, StateVersion}; use tari_ootle_storage::{ - consensus_models::{EpochCheckpoint, SubstateUpdateProof}, + consensus_models::{EpochCheckpoint, SubstateData, SubstateUpdateProof}, StorageError, }; use tari_ootle_storage_sqlite::{error::SqliteStorageError, SqliteTransaction}; @@ -33,7 +33,7 @@ use tari_transaction::{Transaction, TransactionId}; use crate::{ storage_sqlite::{ - models::{EventRecord, KeyValue, NewScannedBlockId, NewSubstate, SubstateRecord, UtxoUpdateRecord}, + models::{EventRecord, KeyValue, NewScannedBlockId, SubstateRecord, UtxoUpdateRecord}, reader::SqliteStoreReadTransaction, writer::SqliteStoreWriteTransaction, }, @@ -209,7 +209,7 @@ pub trait IndexerStoreWriteTransaction { &mut self, updates: I, ) -> Result<(), StorageError>; - fn upsert_substate(&mut self, new_substate: NewSubstate) -> Result<(), StorageError>; + fn upsert_substate(&mut self, substate: &SubstateData) -> Result<(), StorageError>; fn batch_insert_events>(&mut self, events: I) -> Result<(), StorageError>; fn save_scanned_block_id(&mut self, new_scanned_block_id: NewScannedBlockId) -> Result<(), StorageError>; fn delete_scanned_epochs_older_than(&mut self, epoch: Epoch) -> Result<(), StorageError>; diff --git a/applications/tari_indexer/src/storage_sqlite/writer.rs b/applications/tari_indexer/src/storage_sqlite/writer.rs index cfa3fb883c..4f932cb5d5 100644 --- a/applications/tari_indexer/src/storage_sqlite/writer.rs +++ b/applications/tari_indexer/src/storage_sqlite/writer.rs @@ -9,7 +9,7 @@ use serde::Serialize; use tari_engine_types::events::Event; use tari_ootle_common_types::{shard::Shard, substate_type::SubstateType, Epoch, StateVersion}; use tari_ootle_storage::{ - consensus_models::{EpochCheckpoint, SubstateUpdateProof}, + consensus_models::{EpochCheckpoint, SubstateData, SubstateUpdateProof}, StorageError, }; use tari_ootle_storage_sqlite::SqliteTransaction; @@ -170,9 +170,32 @@ impl IndexerStoreWriteTransaction for SqliteStoreWriteTransaction<'_> { Ok(()) } - fn upsert_substate(&mut self, new_substate: NewSubstate) -> Result<(), StorageError> { + fn upsert_substate(&mut self, substate: &SubstateData) -> Result<(), StorageError> { use crate::storage_sqlite::schema::substates; + let template_address = substate + .value + .value() + .and_then(|s| s.component()) + .map(|c| c.template_address.to_string()); + let module_name = substate + .value + .value() + .and_then(|s| s.component()) + .map(|c| c.module_name.clone()); + let new_substate = NewSubstate { + address: substate.substate_id.to_string(), + version: substate.version as i32, + data: substate + .value + .value() + .map(serialize_json) + .transpose()? + .unwrap_or_default(), + template_address, + module_name, + }; + let address = &new_substate.address; let current_substate = substates::table .filter(substates::address.eq(address)) diff --git a/applications/tari_indexer/web_ui/package.json b/applications/tari_indexer/web_ui/package.json index 1695f3b28e..11b4e31516 100644 --- a/applications/tari_indexer/web_ui/package.json +++ b/applications/tari_indexer/web_ui/package.json @@ -7,7 +7,7 @@ "dev": "vite", "build": "tsc && vite build", "preview": "vite preview", - "clean-dist": "rm -rf dist && rm -f tsconfig.tsbuildinfo" + "clean-dist": "rm -rf dist/* && rm -f tsconfig.tsbuildinfo" }, "dependencies": { "@emotion/react": "^11.14.0", diff --git a/applications/tari_swarm_daemon/src/process_definitions/wallet_daemon_create_key.rs b/applications/tari_swarm_daemon/src/process_definitions/wallet_daemon_create_key.rs index a320991189..af1a5af153 100644 --- a/applications/tari_swarm_daemon/src/process_definitions/wallet_daemon_create_key.rs +++ b/applications/tari_swarm_daemon/src/process_definitions/wallet_daemon_create_key.rs @@ -29,19 +29,7 @@ impl ProcessDefinition for WalletDaemonCreateAccount { .arg("-b") .arg(context.base_path()) .arg("--network") - .arg(context.network().to_string()) - .args([ - "create-account", - "--name", - "Fees", - "--key", - "0", - "--set-active", - "--output", - output_path - .to_str() - .context("Non-UTF8 output path in WalletDaemonCreateAccount")?, - ]); + .arg(context.network().to_string()); if let Some(override_keyring_password) = context.get_setting(wallet_daemon::OVERRIDE_KEYRING_PASSWORD_SETTINGS_KEY) @@ -51,6 +39,19 @@ impl ProcessDefinition for WalletDaemonCreateAccount { .arg(override_keyring_password); } + command.args([ + "create-account", + "--name", + "Validator Fees", + "--key", + "0", + "--set-active", + "--output", + output_path + .to_str() + .context("Non-UTF8 output path in WalletDaemonCreateAccount")?, + ]); + Ok(command) } diff --git a/applications/tari_validator_node/src/json_rpc/handlers.rs b/applications/tari_validator_node/src/json_rpc/handlers.rs index b61c422322..9f3ce69810 100644 --- a/applications/tari_validator_node/src/json_rpc/handlers.rs +++ b/applications/tari_validator_node/src/json_rpc/handlers.rs @@ -164,6 +164,7 @@ impl JsonRpcHandlers { .get_local_peer_info() .await .map_err(internal_error(answer_id))?; + let fee_claim_public_key = self.config.validator_node.fee_claim_public_key.to_byte_type(); let response = GetIdentityResponse { peer_id: info.peer_id.to_string(), public_key: self.keypair.public_key().to_byte_type(), @@ -171,6 +172,7 @@ impl JsonRpcHandlers { supported_protocols: info.protocols.into_iter().map(|p| p.to_string()).collect(), protocol_version: info.protocol_version, user_agent: info.agent_version, + fee_claim_public_key, }; Ok(JsonRpcResponse::success(answer_id, response)) diff --git a/applications/tari_validator_node/src/p2p/rpc/block_sync_task.rs b/applications/tari_validator_node/src/p2p/rpc/block_sync_task.rs index c781b2b256..2163c2b4c2 100644 --- a/applications/tari_validator_node/src/p2p/rpc/block_sync_task.rs +++ b/applications/tari_validator_node/src/p2p/rpc/block_sync_task.rs @@ -182,6 +182,7 @@ impl BlockSyncTask { .then(|| child.get_substate_updates(tx, self.num_preshards)) .transpose()? .unwrap_or_default(); + let transaction_receipts = matches!( substates_selection, proto::rpc::StreamSubstateSelection::TransactionReceiptsOnly diff --git a/applications/tari_validator_node/web_ui/package.json b/applications/tari_validator_node/web_ui/package.json index d8bbfd5325..b5ecfe2e4c 100644 --- a/applications/tari_validator_node/web_ui/package.json +++ b/applications/tari_validator_node/web_ui/package.json @@ -7,7 +7,7 @@ "dev": "vite", "build": "tsc && vite build", "preview": "vite preview", - "clean-dist": "rm -rf dist && rm -f tsconfig.tsbuildinfo" + "clean-dist": "rm -rf dist/* && rm -f tsconfig.tsbuildinfo" }, "engines": { "node": ">=18.0.0" diff --git a/applications/tari_validator_node/web_ui/src/routes/VN/Components/Info.tsx b/applications/tari_validator_node/web_ui/src/routes/VN/Components/Info.tsx index f450022fd2..c6da4affcc 100644 --- a/applications/tari_validator_node/web_ui/src/routes/VN/Components/Info.tsx +++ b/applications/tari_validator_node/web_ui/src/routes/VN/Components/Info.tsx @@ -35,58 +35,14 @@ import type { } from "@tari-project/typescript-bindings"; function Info({ - epoch, - identity, - shardKey, -}: { + epoch, + identity, + shardKey, + }: { epoch: GetEpochManagerStatsResponse; identity: VNGetIdentityResponse; shardKey: string | null; }) { - const [registering, setRegistering] = useState(false); - const [registerMessage, setRegisterMessage] = useState(""); - const [feeClaimPublicKey, setRegisterFeeClaimPublicKey] = useState(""); - - const renderShardKey = () => { - if (shardKey === null) - return ( - <> - {/* - Shard key - - {} : register} - > - Register - - {registerMessage ? {registerMessage} : null} - - */} - - Shard key - - setRegisterFeeClaimPublicKey(e.target.value)} - /> - {registerMessage ? {registerMessage} : null} - - - - ); - return ( - - Shard key - {shardKey} - - ); - }; return (
@@ -106,11 +62,14 @@ function Info({ Listen addresses {identity.public_addresses?.join("\n")} + Public key {identity.public_key} + + Claim key {identity.fee_claim_public_key} + - Public key - {identity.public_key} + Shard key + {shardKey} - {renderShardKey()} diff --git a/applications/tari_walletd/src/handlers/validator.rs b/applications/tari_walletd/src/handlers/validator.rs index 9cbe6120ed..6be90d4ca2 100644 --- a/applications/tari_walletd/src/handlers/validator.rs +++ b/applications/tari_walletd/src/handlers/validator.rs @@ -9,7 +9,7 @@ use either::Either; use log::*; use tari_crypto::{keys::PublicKey as _, ristretto::RistrettoPublicKey}; use tari_engine_types::{substate::SubstateId, ToByteType}; -use tari_ootle_common_types::{derive_fee_pool_address, optional::Optional, SubstateRequirement}; +use tari_ootle_common_types::{derive_fee_pool_address, SubstateAddress, SubstateRequirement}; use tari_transaction::args; use tari_wallet_daemon_client::{ permissions::JrpcPermission, @@ -49,41 +49,45 @@ pub async fn handle_get_validator_fees( }, AccountOrKeyIndex::KeyIndex(index) => sdk.key_manager_api().derive_account_key(index)?, }; - let claim_public_key = RistrettoPublicKey::from_secret_key(&claim_key.key); + let claim_public_key = RistrettoPublicKey::from_secret_key(&claim_key.key).to_byte_type(); let shards = req .shard_group .map(|sg| Either::Left(sg.shard_iter())) .unwrap_or_else(|| Either::Right(NUM_PRESHARDS.all_shards_iter())); - let addresses = shards.into_iter().map(|shard| { - ( - shard, - derive_fee_pool_address(&claim_public_key.to_byte_type(), NUM_PRESHARDS, shard), - ) - }); - - let mut fees = HashMap::new(); - - // TODO(perf); bulk scan - for (shard, address) in addresses { - let Some(result) = context + let ids = shards + .into_iter() + .map(|shard| derive_fee_pool_address(&claim_public_key, NUM_PRESHARDS, shard)) + .map(SubstateId::from) + .collect::>(); + + let mut fees = HashMap::with_capacity(ids.len()); + const CHUNK_SIZE: usize = 20; + for id_chunk in ids.chunks(CHUNK_SIZE) { + let substates = context .wallet_sdk() .substate_api() - .fetch_substate_from_network(&SubstateId::from(address), None) - .await - .optional()? - else { - continue; - }; - - let Some(amount) = result.substate.as_validator_fee_pool().map(|p| p.amount()) else { - warn!(target: LOG_TARGET, "Incorrect substate type found at address {}", address); - continue; - }; - - if amount > 0 { - fees.insert(shard, FeePoolDetails { amount, address }); + .get_substates_from_network(id_chunk.to_vec()) + .await?; + + info!(target: LOG_TARGET, "🔍️ Found {}/{} fee pool substates for claim key {}", substates.len(), CHUNK_SIZE, claim_public_key); + + for (substate_id, substate) in substates { + let Some(address) = substate_id.as_validator_fee_pool_address() else { + warn!(target: LOG_TARGET, "Incorrect substate ID found: {}", substate_id); + continue; + }; + + let Some(amount) = substate.substate_value().as_validator_fee_pool().map(|p| p.amount()) else { + warn!(target: LOG_TARGET, "Incorrect substate type found at address {}", substate_id); + continue; + }; + + if amount > 0 { + let shard = SubstateAddress::from_substate_id(&substate_id, substate.version()).to_shard(NUM_PRESHARDS); + fees.insert(shard, FeePoolDetails { amount, address }); + } } } diff --git a/applications/tari_walletd/web_ui/package.json b/applications/tari_walletd/web_ui/package.json index 492b2d841d..3b1c948b1d 100644 --- a/applications/tari_walletd/web_ui/package.json +++ b/applications/tari_walletd/web_ui/package.json @@ -7,7 +7,7 @@ "dev": "vite", "build": "tsc && vite build", "preview": "vite preview", - "clean-dist": "rm -rf dist && rm -f tsconfig.tsbuildinfo" + "clean-dist": "rm -rf dist/* && rm -f tsconfig.tsbuildinfo" }, "dependencies": { "@emotion/react": "^11.14.0", diff --git a/applications/tari_walletd/web_ui/src/utils/helpers.tsx b/applications/tari_walletd/web_ui/src/utils/helpers.tsx index db21e47ccc..209e274897 100644 --- a/applications/tari_walletd/web_ui/src/utils/helpers.tsx +++ b/applications/tari_walletd/web_ui/src/utils/helpers.tsx @@ -61,6 +61,7 @@ export const renderJson = (json: any) => { if (typeof json === "string") return "{json}"; if (typeof json === "number") return {json}; + if (typeof json === "boolean") return {String(json)}; return {json || "null"}; }; diff --git a/bindings/src/types/validator-node-client/VNGetIdentityResponse.ts b/bindings/src/types/validator-node-client/VNGetIdentityResponse.ts index 09694385d0..39287dfdec 100644 --- a/bindings/src/types/validator-node-client/VNGetIdentityResponse.ts +++ b/bindings/src/types/validator-node-client/VNGetIdentityResponse.ts @@ -8,4 +8,5 @@ export type VNGetIdentityResponse = { supported_protocols: Array; protocol_version: string; user_agent: string; + fee_claim_public_key: RistrettoPublicKeyBytes; }; diff --git a/clients/validator_node_client/src/types.rs b/clients/validator_node_client/src/types.rs index e1c8a03fd1..ddbf9e6091 100644 --- a/clients/validator_node_client/src/types.rs +++ b/clients/validator_node_client/src/types.rs @@ -65,6 +65,7 @@ pub struct GetIdentityResponse { pub supported_protocols: Vec, pub protocol_version: String, pub user_agent: String, + pub fee_claim_public_key: RistrettoPublicKeyBytes, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/engine_types/src/substate.rs b/crates/engine_types/src/substate.rs index af70d51e93..42577436f5 100644 --- a/crates/engine_types/src/substate.rs +++ b/crates/engine_types/src/substate.rs @@ -42,7 +42,7 @@ use tari_template_lib::{ VaultId, }, prelude::PUBLIC_IDENTITY_RESOURCE_ADDRESS, - types::{Hash, ObjectKey}, + types::{Hash, ObjectKey, TemplateAddress}, }; use crate::{ @@ -776,6 +776,10 @@ impl SubstateValue { } } + pub fn related_template_address(&self) -> Option { + self.as_component().map(|c| c.template_address) + } + pub fn to_bytes(&self) -> Vec { encode(self).unwrap() } diff --git a/crates/state_store_rocksdb/src/options.rs b/crates/state_store_rocksdb/src/options.rs index 6125c9dac1..67da6ba682 100644 --- a/crates/state_store_rocksdb/src/options.rs +++ b/crates/state_store_rocksdb/src/options.rs @@ -12,7 +12,8 @@ pub struct DatabaseOptions { pub state_history_length: u64, /// The number of epochs back from the current epoch to keep in the database. /// This includes blocks, foreign proposals etc. - /// The default is 1, which means we keep the previous epoch's data. It is not recommended to set this to 0. + /// The default is 1, which means we keep the previous epoch's data until this epoch has passed. It is not + /// recommended to set this to 0. pub epoch_history_length: Epoch, } diff --git a/crates/storage/src/consensus_models/block.rs b/crates/storage/src/consensus_models/block.rs index b18f78de51..e22aa45618 100644 --- a/crates/storage/src/consensus_models/block.rs +++ b/crates/storage/src/consensus_models/block.rs @@ -553,10 +553,6 @@ impl Block { Ok(()) } - pub fn remove_diff(&self, tx: &mut TTx) -> Result<(), StorageError> { - tx.block_diffs_remove(self.id()) - } - pub fn remove_pending_tree_diff_and_return( &self, tx: &mut TTx, @@ -766,6 +762,9 @@ impl Block { tx: &TTx, num_preshards: NumPreshards, ) -> Result, StorageError> { + // The block diff is removed as soon as it is committed, so we need to reconstruct the substate updates from the + // committed transactions. TODO: this does not include "implicit" state transitions i.e. validator fee + // pools. let committed = self .commands() .iter() diff --git a/crates/storage/src/consensus_models/command.rs b/crates/storage/src/consensus_models/command.rs index e44bdfcf0a..0ba3d49e3e 100644 --- a/crates/storage/src/consensus_models/command.rs +++ b/crates/storage/src/consensus_models/command.rs @@ -86,7 +86,6 @@ pub enum Command { #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] enum CommandOrdering<'a> { EvictNode, - // EvictNode, /// Foreign proposals should come first in the block so that they are processed before commands ForeignProposal(ShardGroup, &'a BlockId), TransactionId(&'a TransactionId), diff --git a/crates/wallet/sdk/src/apis/accounts.rs b/crates/wallet/sdk/src/apis/accounts.rs index 37c505577b..3dc5ad6686 100644 --- a/crates/wallet/sdk/src/apis/accounts.rs +++ b/crates/wallet/sdk/src/apis/accounts.rs @@ -22,7 +22,7 @@ use tari_template_lib::{ use crate::{ apis::{ confidential_transfer::{ConfidentialTransferApiError, ResolvedAccountDetails}, - key_manager::{KeyManagerApi, KeyManagerApiError}, + key_manager::{KeyBranch, KeyManagerApi, KeyManagerApiError}, substate::{SubstatesApi, ValidatorScanResult}, }, models::{Account, AccountUpdate, AccountWithAddress, VaultBalance, VaultModel}, @@ -111,6 +111,7 @@ impl<'a, TStore: WalletStore, TNetworkInterface> AccountsApi<'a, TStore, TNetwor return Err(AccountsApiError::AccountNameAlreadyExists { name: name.to_string() }); } } + tx.key_manager_insert_or_ignore(KeyBranch::Account.as_str(), owner_key_index)?; tx.accounts_insert( account_name, account_address, diff --git a/crates/wallet/sdk/src/apis/substate.rs b/crates/wallet/sdk/src/apis/substate.rs index 8ebed51e37..b9925f49e9 100644 --- a/crates/wallet/sdk/src/apis/substate.rs +++ b/crates/wallet/sdk/src/apis/substate.rs @@ -1,13 +1,13 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use log::*; use tari_engine_types::{ indexed_value::{IndexedValueError, IndexedWellKnownTypes}, resource::Resource, - substate::{SubstateId, SubstateValue}, + substate::{Substate, SubstateId, SubstateValue}, transaction_receipt::TransactionReceiptAddress, }; use tari_ootle_common_types::{ @@ -63,6 +63,16 @@ where Ok(substates) } + pub async fn get_substates_from_network( + &self, + ids: Vec, + ) -> Result, SubstateApiError> { + self.network_interface + .get_substates(ids) + .await + .map_err(|e| SubstateApiError::NetworkInterfaceError(e.into())) + } + pub fn load_dependent_substates( &self, parents: &[&SubstateId], diff --git a/crates/wallet/storage_sqlite/src/writer.rs b/crates/wallet/storage_sqlite/src/writer.rs index 7452c395b9..27a9add6a4 100644 --- a/crates/wallet/storage_sqlite/src/writer.rs +++ b/crates/wallet/storage_sqlite/src/writer.rs @@ -221,6 +221,9 @@ impl WalletStoreWriter for WriteTransaction<'_> { let index = i64::try_from(index) .map_err(|_| WalletStorageError::general("key_manager_set_active_index", "index too large"))?; + // Ensure it exists + self.key_manager_insert_or_ignore(branch, index as u64)?; + let active_id = key_manager_states::table .select(key_manager_states::id) .filter(key_manager_states::branch_seed.eq(branch))