diff --git a/.github/workflows/npm_publish.yml b/.github/workflows/npm_publish.yml index e6d948a94f..95a4a4f54d 100644 --- a/.github/workflows/npm_publish.yml +++ b/.github/workflows/npm_publish.yml @@ -67,4 +67,34 @@ jobs: pnpm publish --access public --no-git-checks || echo "Publish failed (probably already published)" env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - + + + # Publish the Tari Indexer client to the NPM registry + publish-indexer-client: + runs-on: ubuntu-latest + needs: publish-typescript-bindings + defaults: + run: + working-directory: clients/javascript/indexer_client + steps: + - uses: actions/checkout@v5 + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: ${{ env.PNPM_VERSION }} + - name: Use Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@v5 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'pnpm' + - run: pnpm install --frozen-lockfile + - run: pnpm run build-dist + name: "Build Bindings Dist" + working-directory: bindings + - run: pnpm run build + - run: | + echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc + pnpm publish --access public --no-git-checks || echo "Publish failed (probably already published)" + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + diff --git a/Cargo.lock b/Cargo.lock index d81da0778d..8d8ad59395 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -465,7 +465,7 @@ dependencies = [ "serde_json", "tokio", "tokio-stream", - "tokio-util 0.7.15", + "tokio-util 0.7.16", "tower-service", ] @@ -3910,7 +3910,7 @@ dependencies = [ "indexmap 2.11.4", "slab", "tokio", - "tokio-util 0.7.15", + "tokio-util 0.7.16", "tracing", ] @@ -3929,7 +3929,7 @@ dependencies = [ "indexmap 2.11.4", "slab", "tokio", - "tokio-util 0.7.15", + "tokio-util 0.7.16", "tracing", ] @@ -4045,6 +4045,17 @@ dependencies = [ "sha1 0.10.6", ] +[[package]] +name = "headers-accept" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479bcb872e714e11f72fcc6a71afadbc86d0dbe887bc44252b04cfbc63272897" +dependencies = [ + "headers-core", + "http 1.3.1", + "mediatype", +] + [[package]] name = "headers-core" version = "0.3.0" @@ -4135,7 +4146,7 @@ dependencies = [ "once_cell", "rand 0.9.1", "ring 0.17.14", - "socket2", + "socket2 0.5.10", "thiserror 2.0.12", "tinyvec", "tokio", @@ -4414,7 +4425,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -4532,7 +4543,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.5.10", "system-configuration 0.6.1", "tokio", "tower-service", @@ -5048,13 +5059,24 @@ dependencies = [ "rustversion", ] +[[package]] +name = "io-uring" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +dependencies = [ + "bitflags 2.9.2", + "cfg-if", + "libc", +] + [[package]] name = "ipconfig" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2", + "socket2 0.5.10", "widestring", "windows-sys 0.48.0", "winreg", @@ -5711,7 +5733,7 @@ dependencies = [ "libp2p-swarm", "rand 0.8.5", "smallvec 1.15.1", - "socket2", + "socket2 0.5.10", "tokio", "tracing", ] @@ -5828,7 +5850,7 @@ dependencies = [ "rand 0.8.5", "ring 0.17.14", "rustls 0.23.28", - "socket2", + "socket2 0.5.10", "thiserror 2.0.12", "tokio", "tracing", @@ -5946,7 +5968,7 @@ dependencies = [ "if-watch", "libc", "libp2p-core", - "socket2", + "socket2 0.5.10", "tokio", "tracing", ] @@ -6367,6 +6389,12 @@ dependencies = [ "digest", ] +[[package]] +name = "mediatype" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f490ea2ae935dd8ac89c472d4df28c7f6b87cc20767e1b21fd5ed6a16e7f61e4" + [[package]] name = "memchr" version = "2.7.5" @@ -8573,7 +8601,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.28", - "socket2", + "socket2 0.5.10", "thiserror 2.0.12", "tokio", "tracing", @@ -8610,7 +8638,7 @@ dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2", + "socket2 0.5.10", "tracing", "windows-sys 0.59.0", ] @@ -8988,10 +9016,12 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", + "tokio-util 0.7.16", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots 0.25.4", "winreg", @@ -10215,6 +10245,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -11144,22 +11184,25 @@ dependencies = [ "async-graphql", "async-graphql-axum", "axum 0.8.4", - "axum-jrpc", "bincode 2.0.1", + "bytes 1.10.1", "cacache", "clap 3.2.25", "config", "diesel", "diesel_migrations", "futures 0.3.31", + "headers-accept", "hex", "include_dir", "indexmap 2.11.4", "libp2p", "log", "log4rs", + "mediatype", "mime_guess", "minotari_app_utilities", + "prost 0.14.1", "serde", "serde_json", "serde_with", @@ -11194,6 +11237,8 @@ dependencies = [ "tokio", "tower-http", "url", + "utoipa", + "utoipa-swagger-ui", ] [[package]] @@ -11202,11 +11247,15 @@ version = "0.11.2" dependencies = [ "anyhow", "bounded-vec", + "bytes 1.10.1", + "futures 0.3.31", + "log", "multiaddr 0.18.1", + "prost 0.14.1", "reqwest 0.11.27", "serde", "serde_json", - "serde_with", + "serde_urlencoded", "tari_common_types", "tari_consensus_types", "tari_engine_types", @@ -11217,7 +11266,9 @@ dependencies = [ "tari_template_lib_types", "tari_transaction", "thiserror 1.0.69", + "tokio", "ts-rs", + "utoipa", ] [[package]] @@ -11542,6 +11593,7 @@ dependencies = [ "anyhow", "blake2", "digest", + "futures 0.3.31", "keyring", "log", "passwords", @@ -11573,6 +11625,7 @@ name = "tari_ootle_wallet_sdk_services" version = "0.11.2" dependencies = [ "anyhow", + "futures 0.3.31", "futures-bounded", "log", "reqwest 0.11.27", @@ -11720,7 +11773,7 @@ dependencies = [ "tari_shutdown", "thiserror 1.0.69", "tokio", - "tokio-util 0.7.15", + "tokio-util 0.7.16", "tower 0.5.2", "tracing", ] @@ -12621,21 +12674,23 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.1" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes 1.10.1", + "io-uring", "libc", "mio 1.0.4", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "slab", + "socket2 0.6.0", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -12688,7 +12743,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", - "tokio-util 0.7.15", + "tokio-util 0.7.16", ] [[package]] @@ -12720,9 +12775,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.15" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" dependencies = [ "bytes 1.10.1", "futures-core", @@ -12842,7 +12897,7 @@ dependencies = [ "percent-encoding", "pin-project 1.1.10", "prost 0.13.5", - "socket2", + "socket2 0.5.10", "tokio", "tokio-stream", "tower 0.4.13", @@ -12872,7 +12927,7 @@ dependencies = [ "pin-project 1.1.10", "prost 0.13.5", "rustls-native-certs", - "socket2", + "socket2 0.5.10", "tokio", "tokio-rustls 0.26.2", "tokio-stream", @@ -12924,7 +12979,7 @@ dependencies = [ "rand 0.8.5", "slab", "tokio", - "tokio-util 0.7.15", + "tokio-util 0.7.16", "tower-layer", "tower-service", "tracing", @@ -12943,7 +12998,7 @@ dependencies = [ "slab", "sync_wrapper 1.0.2", "tokio", - "tokio-util 0.7.15", + "tokio-util 0.7.16", "tower-layer", "tower-service", "tracing", @@ -12970,7 +13025,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "tokio", - "tokio-util 0.7.15", + "tokio-util 0.7.16", "tower 0.5.2", "tower-layer", "tower-service", @@ -13166,7 +13221,7 @@ dependencies = [ "stun", "thiserror 1.0.69", "tokio", - "tokio-util 0.7.15", + "tokio-util 0.7.16", "webrtc-util", ] @@ -13668,6 +13723,19 @@ dependencies = [ "cxx-build", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmer" version = "6.0.1" @@ -14120,7 +14188,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce981f93104a8debb3563bb0cedfe4aa2f351fdf6b53f346ab50009424125c08" dependencies = [ "log", - "socket2", + "socket2 0.5.10", "thiserror 1.0.69", "tokio", "webrtc-util", diff --git a/Cargo.toml b/Cargo.toml index f03e95f2b7..c708c054ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -238,9 +238,9 @@ syn = "2.0" tempfile = "3.3.0" thiserror = "1.0.59" time = "0.3.15" -tokio = { version = "1.44.1", default-features = false } -tokio-stream = "0.1.7" -tokio-util = "0.7.10" +tokio = { version = "1.47.1", default-features = false } +tokio-stream = "0.1.17" +tokio-util = "0.7.16" tonic = "0.13.1" tower = "0.5" tower-http = { version = "0.6", default-features = false } @@ -255,6 +255,8 @@ webrtc = "0.9.0" zeroize = "1" webauthn-rs = { version = "0.5.1", features = ["default"] } webauthn-rs-proto = { version = "0.5.1", features = ["default"] } +utoipa = "5.4.0" +utoipa-swagger-ui = { version = "9.0.2" } uuid = "1.15.0" # Shutdown when panicking so we can see the error, specifically for the wallet diff --git a/applications/tari_app_utilities/config_presets/d_indexer.toml b/applications/tari_app_utilities/config_presets/d_indexer.toml index 7869a6add0..169760ba7b 100644 --- a/applications/tari_app_utilities/config_presets/d_indexer.toml +++ b/applications/tari_app_utilities/config_presets/d_indexer.toml @@ -22,8 +22,8 @@ # The relative path to store persistent data (default = "data/indexer") #data_dir = "data/indexer" -# JSON-RPC listener address (default = "127.0.0.1:18300") -#json_rpc_address = "127.0.0.1:18300" +# API listener address (default = "127.0.0.1:18300") +#api_listen_address = "127.0.0.1:18300" # HTTP UI listener address (default = "127.0.0.1:15000") #web_ui_address = "127.0.0.1:15000" diff --git a/applications/tari_app_utilities/config_presets/e_wallet_daemon.toml b/applications/tari_app_utilities/config_presets/e_wallet_daemon.toml index fee8d80205..d0f381f886 100644 --- a/applications/tari_app_utilities/config_presets/e_wallet_daemon.toml +++ b/applications/tari_app_utilities/config_presets/e_wallet_daemon.toml @@ -12,8 +12,8 @@ # signaling_server_addr = "127.0.0.1:9100" [localnet.ootle_wallet_daemon] -# Indexer JSON-RPC url (default: http://127.0.0.1:18200/json_rpc) -# indexer_json_rpc_url = "http://127.0.0.1:18200/json_rpc" +# Indexer API url (default: http://127.0.0.1:18200) +# indexer_api_url = "http://127.0.0.1:18200" [igor.ootle_wallet_daemon] -# Indexer JSON-RPC url for Igor. -indexer_json_rpc_url = "http://18.217.22.26:12500/json_rpc" +# Indexer API url for Igor. +indexer_api_url = "http://18.217.22.26:12500" diff --git a/applications/tari_indexer/Cargo.toml b/applications/tari_indexer/Cargo.toml index dcdee687bf..9fe6618f0e 100644 --- a/applications/tari_indexer/Cargo.toml +++ b/applications/tari_indexer/Cargo.toml @@ -20,7 +20,7 @@ tari_engine = { workspace = true } tari_engine_types = { workspace = true } tari_epoch_manager = { workspace = true } tari_epoch_oracles = { workspace = true, features = ["base_layer"] } -tari_indexer_client = { workspace = true, default-features = false } +tari_indexer_client = { workspace = true, default-features = false, features = ["utoipa"] } tari_indexer_lib = { workspace = true } tari_networking = { workspace = true } tari_ootle_app_utilities = { workspace = true } @@ -37,10 +37,10 @@ tari_validator_node_rpc = { workspace = true } tari_ootle_wallet_sdk = { workspace = true } anyhow = { workspace = true } -axum = { workspace = true } +axum = { workspace = true, features = ["query", "matched-path"] } async-graphql = { workspace = true } async-graphql-axum = { workspace = true } -axum-jrpc = { workspace = true, features = ["anyhow_error"] } +bytes = { workspace = true } bincode = { workspace = true, features = ["serde"] } clap = { workspace = true, features = ["derive", "env"] } config = { workspace = true } @@ -56,6 +56,7 @@ include_dir = { workspace = true, optional = true } libp2p = { workspace = true } log = { workspace = true, features = ["std"] } hex = { workspace = true } +headers-accept = "0.2.1" log4rs = { workspace = true, features = [ "rolling_file_appender", "compound_policy", @@ -63,6 +64,7 @@ log4rs = { workspace = true, features = [ "fixed_window_roller", ] } mime_guess = { workspace = true } +mediatype = "0.20.0" serde = { workspace = true, features = ["default", "derive"] } serde_json = { workspace = true } serde_with = { workspace = true, features = ["indexmap_2"] } @@ -75,9 +77,12 @@ tokio = { workspace = true, features = [ "sync", "rt-multi-thread", ] } -tower-http = { workspace = true, features = ["default", "cors"] } +tower-http = { workspace = true, features = ["default", "cors", "limit"] } url = { workspace = true, features = ["serde"] } cacache = { version = "13.1.0", default-features = false, features = ["mmap", "tokio-runtime"] } +prost = { workspace = true } +utoipa = { workspace = true } +utoipa-swagger-ui = { workspace = true, features = ["axum"] } [package.metadata.cargo-machete] ignored = [ diff --git a/applications/tari_indexer/README.md b/applications/tari_indexer/README.md new file mode 100644 index 0000000000..15917e0959 --- /dev/null +++ b/applications/tari_indexer/README.md @@ -0,0 +1,137 @@ +# Tari Indexer + +The Tari Indexer is a service that indexes and tracks substates, transactions, and events on the Tari DAN (Digital Assets Network). It provides REST and GraphQL APIs for querying blockchain data and includes a web-based user interface for exploring the network. + +## Overview + +The indexer continuously scans the DAN layer for new blocks, transactions, and substates, storing this information in a local SQLite database. It provides multiple interfaces for accessing this data: + +- **REST API**: RESTful endpoints for programmatic access +- **GraphQL API**: Flexible query interface for complex data retrieval +- **Web UI**: Browser-based interface for exploring network data +- **UTXO Streaming**: Real-time streaming of UTXO updates + +## Features + +- Real-time blockchain data indexing +- REST and GraphQL APIs +- Web-based user interface +- UTXO streaming capabilities +- P2P networking for validator node communication +- Configurable scanning intervals +- Template and NFT tracking +- Transaction and event monitoring + +## Building + +```bash +cargo build --release +``` + +### Features + +- `web_ui` (default): Includes the web-based user interface +- Without web UI: `cargo build --release --no-default-features` + +## Configuration + +The indexer uses configuration files and environment variables. Key configuration sections: + +- **API Endpoints**: Configure REST and GraphQL server addresses +- **P2P Networking**: Set peer seeds, ports, and reachability modes +- **Database**: SQLite storage configuration +- **Scanning**: Block scanning intervals and filters +- **Web UI**: Public API URLs for the web interface + +## Running + +### Basic Usage + +```bash +./target/release/tari_indexer +``` + +### Command Line Options + +- `-r, --api-listen-address `: Bind address for REST API server +- `-g, --node-grpc `: Minotari node gRPC URL for epoch oracle +- `-s, --peer-seeds `: P2P peer seeds (space-separated) +- `--listener-port `: P2P listening port +- `-i, --scanning-interval `: Block scanning interval +- `--web-ui-public-api-url `: Public API URL for web UI +- `--web-ui-public-graphql-url `: Public GraphQL URL for web UI +- `--reachability `: P2P reachability mode (reachable/unreachable) +- `--disable-mdns`: Disable mDNS peer discovery + +## APIs + +### REST API + +When running, the REST API is available at the configured address (default: `http://localhost:8080`). Key endpoints include: + +- `/templates` - Template information +- `/transactions` - Transaction data +- `/substates` - Substate queries +- `/nfts` - NFT tracking +- `/utxos/stream` - UTXO streaming +- `/network` - Network information + +Swagger documentation is available at `/swagger-ui/` when the server is running. + +### GraphQL API + +The GraphQL endpoint provides flexible querying capabilities for: +- Transactions and their details +- Substates and their changes +- Events and logs +- Template information + +### Web UI + +The web interface (when enabled) provides: +- Network overview and statistics +- Transaction browsing and search +- Substate exploration +- Event monitoring +- Resource and NFT tracking + +## Database + +The indexer uses SQLite for local storage, automatically creating and migrating the database schema on startup. The database location is configurable through the data directory setting. + +## Development + +### Web UI Development + +The web UI is a React application located in the `web_ui/` directory: + +```bash +cd web_ui/ +npm install +npm run dev # Development server +npm run build # Production build +``` + +### Testing + +Run the test suite: + +```bash +cargo test +``` + +## Architecture + +The indexer consists of several key components: + +- **Block Scanner**: Continuously scans for new blocks and events +- **Storage Layer**: SQLite-based data persistence +- **REST Server**: HTTP API endpoints +- **GraphQL Server**: Flexible query interface +- **P2P Networking**: Communication with validator nodes +- **Event Manager**: Real-time event processing +- **Web UI Server**: Static file serving for the web interface + +## License + +Copyright 2023, The Tari Project. Licensed under BSD 3-Clause. \ No newline at end of file diff --git a/applications/tari_indexer/src/bootstrap.rs b/applications/tari_indexer/src/bootstrap.rs index 7359b724b7..0dd486e9a9 100644 --- a/applications/tari_indexer/src/bootstrap.rs +++ b/applications/tari_indexer/src/bootstrap.rs @@ -33,6 +33,7 @@ use tari_common::{ }; use tari_consensus::consensus_constants::ConsensusConstants; use tari_crypto::tari_utilities::ByteArray; +use tari_engine::transaction::TransactionProcessorConfig; use tari_engine_types::ToByteType; use tari_epoch_manager::service::{EpochManagerConfig, EpochManagerHandle}; use tari_epoch_oracles::{ @@ -42,8 +43,10 @@ use tari_epoch_oracles::{ store::EpochOracleStore, EpochOracle, }; +use tari_indexer_lib::substate_scanner::SubstateScanner; use tari_networking::{MessagingMode, NetworkingHandle, RelayCircuitLimits, RelayReservationLimits, SwarmConfig}; use tari_ootle_app_utilities::{ + claim_burn_proof_verifier::TariClaimBurnProofVerifier, common::verify_correct_network, configuration::convert_network_to_l1_network, epoch_oracle_config::EpochOracleType, @@ -61,6 +64,7 @@ use tari_template_manager::{implementation::TemplateManager, interface::Template use tari_validator_node_rpc::client::TariValidatorNodeRpcClientFactory; use crate::{ + dry_run::processor::DryRunTransactionProcessor, network_client::TariNetworkClient, network_state_sync, network_state_sync::NetworkWideStateSyncConfig, @@ -71,6 +75,9 @@ use crate::{ IndexerStoreWriteTransaction, SqliteIndexerStore, }, + substate_file_cache::SubstateFileCache, + substate_manager::SubstateManager, + transaction_manager::TransactionManager, ApplicationConfig, IndexerEpochManagerSpec, Noop, @@ -216,6 +223,29 @@ pub async fn spawn_services( ) .spawn(shutdown.clone()); + let substate_cache_dir = config.common.base_path.join("substate_cache"); + let substate_cache = SubstateFileCache::new(substate_cache_dir) + .map_err(|e| ExitError::new(ExitCode::ConfigError, format!("Substate cache error: {}", e)))?; + + let substate_scanner = SubstateScanner::new( + epoch_manager.clone(), + validator_node_client_factory.clone(), + substate_cache, + ); + + let substate_manager = SubstateManager::new(substate_scanner.clone(), store.clone()); + let transaction_manager = TransactionManager::new(network_client.clone(), store.clone()); + + // dry run + let dry_run_transaction_processor = DryRunTransactionProcessor::new( + TransactionProcessorConfig::new(config.network) + .with_template_binary_max_size_bytes(consensus_constants.template_binary_max_size_bytes), + epoch_manager.clone(), + validator_node_client_factory.clone(), + template_manager.clone(), + TariClaimBurnProofVerifier::new(config.network, global_db.clone()), + ); + // Save final node identity after comms has initialized. This is required because the public_address can be // changed by comms during initialization when using tor. save_identities(config, &keypair)?; @@ -224,11 +254,13 @@ pub async fn spawn_services( networking, epoch_manager, validator_node_client_factory, - network_client, store, global_db, template_manager, _template_manager_service: template_manager_service, + substate_manager, + transaction_manager, + dry_run_transaction_processor, }) } @@ -238,10 +270,13 @@ pub struct Services { pub epoch_manager: EpochManagerHandle, pub validator_node_client_factory: TariValidatorNodeRpcClientFactory, pub store: SqliteIndexerStore, - pub network_client: TariNetworkClient, TariValidatorNodeRpcClientFactory>, pub global_db: GlobalDb>, pub template_manager: TemplateManager, pub _template_manager_service: TemplateManagerHandle, + pub substate_manager: SubstateManager, + pub transaction_manager: + TransactionManager, TariValidatorNodeRpcClientFactory, SqliteIndexerStore>, + pub dry_run_transaction_processor: DryRunTransactionProcessor, } fn ensure_directories_exist(config: &ApplicationConfig) -> io::Result<()> { diff --git a/applications/tari_indexer/src/cli.rs b/applications/tari_indexer/src/cli.rs index 803ba5dccc..02b78e6588 100644 --- a/applications/tari_indexer/src/cli.rs +++ b/applications/tari_indexer/src/cli.rs @@ -43,9 +43,9 @@ pub struct Cli { #[clap(long, short = 'i')] pub scanning_interval: Option, - /// Bind address for JSON-rpc server - #[clap(long, short = 'r', alias = "rpc-address")] - pub json_rpc_address: Option, + /// Bind address for API server + #[clap(long, short = 'r', alias = "api-address")] + pub api_listen_address: Option, #[clap(long, alias = "node-grpc", short = 'g', env = "TARI_INDEXER_MINOTARI_NODE_GRPC_URL")] pub epoch_oracle_minotari_node_grpc_url: Option, #[clap(long, alias = "oracle-config")] @@ -61,9 +61,9 @@ pub struct Cli { pub reachability: Option, #[clap(long)] pub disable_mdns: bool, - /// Public address of the JSON RPC - #[clap(long, env = "TARI_INDEXER_WEB_UI_PUBLIC_JSON_RPC_URL")] - pub web_ui_public_json_rpc_url: Option, + /// Public address of the API + #[clap(long, env = "TARI_INDEXER_WEB_UI_PUBLIC_API_URL")] + pub web_ui_public_api_url: Option, /// Public address of the GraphQL API #[clap(long, env = "TARI_INDEXER_WEB_UI_PUBLIC_GRAPHQL_URL")] pub web_ui_public_graphql_url: Option, @@ -86,8 +86,8 @@ impl ConfigOverrideProvider for Cli { overrides.push(("indexer.override_from".to_string(), network.to_string())); overrides.push(("p2p.seeds.override_from".to_string(), network.to_string())); - if let Some(ref addr) = self.json_rpc_address { - overrides.push(("indexer.json_rpc_address".to_string(), addr.to_string())); + if let Some(ref addr) = self.api_listen_address { + overrides.push(("indexer.api_listen_address".to_string(), addr.to_string())); } if !self.peer_seeds.is_empty() { @@ -99,11 +99,8 @@ impl ConfigOverrideProvider for Cli { if let Some(seconds) = self.scanning_interval { overrides.push(("indexer.scanning_interval".to_string(), seconds.to_string())); } - if let Some(ref json_rpc_url) = self.web_ui_public_json_rpc_url { - overrides.push(( - "indexer.web_ui_public_json_rpc_url".to_string(), - json_rpc_url.to_string(), - )); + if let Some(ref api_url) = self.web_ui_public_api_url { + overrides.push(("indexer.web_ui_public_api_url".to_string(), api_url.to_string())); } if let Some(ref graphql_url) = self.web_ui_public_graphql_url { overrides.push(("indexer.web_ui_public_graphql_url".to_string(), graphql_url.to_string())); diff --git a/applications/tari_indexer/src/config.rs b/applications/tari_indexer/src/config.rs index 4b0bd56965..d89b200323 100644 --- a/applications/tari_indexer/src/config.rs +++ b/applications/tari_indexer/src/config.rs @@ -80,15 +80,15 @@ pub struct IndexerConfig { pub data_dir: PathBuf, /// The p2p configuration settings pub p2p: P2pConfig, - /// JSON-RPC address of the indexer application - pub json_rpc_address: Option, + /// Listening address for the indexer API server + pub api_listen_address: Option, /// GraphQL port of the indexer application pub graphql_address: Option, /// The address of the Web UI pub web_ui_address: Option, - /// The jrpc address where the UI should connect to the JSON RPC (it can be the same as the json_rpc_address, but - /// doesn't have to be), if this will be None, then the listen_addr will be used. - pub web_ui_public_json_rpc_url: Option, + /// The publicly-accessible URL that the UI uses to connect to the API. + /// If this is None, then the api_listen_address will be used. + pub web_ui_public_api_url: Option, /// The jrpc address where the UI should connect to the GraphQL API(it can be the same as the json_rpc_address, but /// doesn't have to be), if this will be None, then the listen_addr will be used. pub web_ui_public_graphql_url: Option, @@ -135,10 +135,10 @@ impl Default for IndexerConfig { tor_identity_file: PathBuf::from("indexer_tor_id.json"), data_dir: PathBuf::from("data/indexer"), p2p: P2pConfig::default(), - json_rpc_address: Some("127.0.0.1:18300".parse().unwrap()), + api_listen_address: Some("127.0.0.1:18300".parse().unwrap()), graphql_address: Some("127.0.0.1:18301".parse().unwrap()), web_ui_address: Some("127.0.0.1:15000".parse().unwrap()), - web_ui_public_json_rpc_url: None, + web_ui_public_api_url: None, web_ui_public_graphql_url: None, block_scanning_interval: Duration::from_secs(10), state_scanning_interval: Duration::from_secs(60), diff --git a/applications/tari_indexer/src/dry_run/processor.rs b/applications/tari_indexer/src/dry_run/processor.rs index dceffa93ae..0db44010bc 100644 --- a/applications/tari_indexer/src/dry_run/processor.rs +++ b/applications/tari_indexer/src/dry_run/processor.rs @@ -49,6 +49,7 @@ use crate::dry_run::error::DryRunTransactionProcessorError; const LOG_TARGET: &str = "tari::indexer::dry_run_transaction_processor"; +#[derive(Clone)] pub struct DryRunTransactionProcessor { config: TransactionProcessorConfig, epoch_manager: EpochManagerHandle, diff --git a/applications/tari_indexer/src/http_ui/server.rs b/applications/tari_indexer/src/http_ui/server.rs index 407e3bc201..65b38cf446 100644 --- a/applications/tari_indexer/src/http_ui/server.rs +++ b/applications/tari_indexer/src/http_ui/server.rs @@ -37,13 +37,13 @@ const LOG_TARGET: &str = "tari::indexer::web_ui::server"; pub async fn run_http_ui_server( address: SocketAddr, - json_rpc_address: Url, + api_address: Url, graphql_address: Option, ) -> Result<(), anyhow::Error> { - let json_rpc_address = Arc::new(json_rpc_address); + let api_address = Arc::new(api_address); let router = Router::new() - .route("/json_rpc_address", get(|| async move { json_rpc_address.to_string() })) + .route("/rest_api_address", get(|| async move { api_address.to_string() })) .route( "/graphql_address", get(|| async move { graphql_address.map(|a| a.to_string()).unwrap_or_default() }), diff --git a/applications/tari_indexer/src/json_rpc/error.rs b/applications/tari_indexer/src/json_rpc/error.rs deleted file mode 100644 index 110fbbf5d8..0000000000 --- a/applications/tari_indexer/src/json_rpc/error.rs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2023 The Tari Project -// SPDX-License-Identifier: BSD-3-Clause - -use std::{env, fmt::Display}; - -use axum_jrpc::{ - error::{JsonRpcError, JsonRpcErrorReason}, - JsonRpcResponse, -}; - -const LOG_TARGET: &str = "tari::indexer::json_rpc"; - -pub fn internal_error(answer_id: axum_jrpc::Id) -> impl Fn(T) -> JsonRpcResponse { - move |err| { - log::error!(target: LOG_TARGET, "🚨 Internal error: {}", err); - let msg = if cfg!(debug_assertions) || - env::var("CI").is_ok() || - env::var("DEBUG_MODE").ok().as_deref() == Some("1") - { - format!("An internal error occurred: {}", err) - } else { - "An internal error occurred".to_string() - }; - JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new(JsonRpcErrorReason::InternalError, msg, serde_json::Value::Null), - ) - } -} diff --git a/applications/tari_indexer/src/json_rpc/handlers.rs b/applications/tari_indexer/src/json_rpc/handlers.rs deleted file mode 100644 index 4d0bfaa879..0000000000 --- a/applications/tari_indexer/src/json_rpc/handlers.rs +++ /dev/null @@ -1,906 +0,0 @@ -// Copyright 2023. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use std::{collections::HashMap, fmt::Display, ops::Deref}; - -use axum_jrpc::{ - error::{JsonRpcError, JsonRpcErrorReason}, - JrpcResult, - JsonRpcExtractor, - JsonRpcResponse, -}; -use libp2p::swarm::dial_opts::{DialOpts, PeerCondition}; -use log::{error, info, warn}; -use serde_json::{self as json, json, Value}; -use tari_consensus_types::Decision; -use tari_crypto::{ristretto::RistrettoPublicKey, tari_utilities::hex::to_hex}; -use tari_engine::{template::TemplateModuleLoader, wasm::WasmModule}; -use tari_engine_types::{ConvertFromByteType, ToByteType}; -use tari_epoch_manager::{service::EpochManagerHandle, EpochManagerReader}; -use tari_epoch_oracles::store::StoreKey; -use tari_indexer_client::types::{ - self, - AddPeerRequest, - AddPeerResponse, - ConnectionDirection, - GetCommsStatsResponse, - GetConnectionsResponse, - GetEpochManagerStatsResponse, - GetIdentityResponse, - GetNetworkSyncStateResponse, - GetNonFungiblesRequest, - GetNonFungiblesResponse, - GetSubstateRequest, - GetSubstateResponse, - GetSubstatesRequest, - GetSubstatesResponse, - GetTemplateDefinitionRequest, - GetTemplateDefinitionResponse, - GetTransactionResultRequest, - GetTransactionResultResponse, - GetUnspentUtxosRequest, - GetUnspentUtxosResponse, - GetUtxoUpdatesRequest, - GetUtxoUpdatesResponse, - IndexerReadyResponse, - IndexerTransactionFinalizedResult, - InspectSubstateRequest, - InspectSubstateResponse, - ListRecentTransactionsRequest, - ListRecentTransactionsResponse, - ListSubstatesRequest, - ListSubstatesResponse, - ListTemplatesRequest, - ListTemplatesResponse, - NetworkDescription, - SubmitTransactionRequest, - SubmitTransactionResponse, - SyncProgress, - TemplateMetadata, -}; -use tari_networking::{is_supported_multiaddr, NetworkingHandle, NetworkingService}; -use tari_ootle_app_utilities::keypair::RistrettoKeypair; -use tari_ootle_common_types::{ - optional::Optional, - public_key_to_peer_id, - NumPreshards, - PeerAddress, - SubstateRequirement, -}; -use tari_ootle_p2p::TariMessagingSpec; -use tari_ootle_storage::{ - global::{GlobalDb, TemplateStatus}, - time::{PrimitiveDateTime, UtcDateTime}, -}; -use tari_ootle_storage_sqlite::global::SqliteGlobalDbAdapter; -use tari_ootle_wallet_sdk::models::{UtxoStateUpdateSet, UtxoUpdateSet}; -use tari_template_manager::{ - implementation::TemplateManager, - interface::{TemplateExecutable, TemplateManagerError}, -}; -use tari_validator_node_rpc::client::{SubstateResult, TariValidatorNodeRpcClientFactory, TransactionResultStatus}; - -use crate::{ - bootstrap::Services, - dry_run::processor::DryRunTransactionProcessor, - json_rpc::error::internal_error, - network_client::NetworkClientError, - storage_sqlite::SqliteIndexerStore, - substate_manager::SubstateManager, - transaction_manager::{error::TransactionManagerError, TransactionManager}, -}; - -const LOG_TARGET: &str = "tari::indexer::json_rpc::handlers"; - -pub struct JsonRpcHandlers { - keypair: RistrettoKeypair, - networking: NetworkingHandle, - substate_manager: SubstateManager, - epoch_manager: EpochManagerHandle, - transaction_manager: - TransactionManager, TariValidatorNodeRpcClientFactory, SqliteIndexerStore>, - template_manager: TemplateManager, - global_db: GlobalDb>, - dry_run_transaction_processor: DryRunTransactionProcessor, -} - -impl JsonRpcHandlers { - pub fn new( - services: &Services, - substate_manager: SubstateManager, - transaction_manager: TransactionManager< - EpochManagerHandle, - TariValidatorNodeRpcClientFactory, - SqliteIndexerStore, - >, - global_db: GlobalDb>, - template_manager: TemplateManager, - dry_run_transaction_processor: DryRunTransactionProcessor, - ) -> Self { - Self { - keypair: services.keypair.clone(), - networking: services.networking.clone(), - global_db, - substate_manager, - epoch_manager: services.epoch_manager.clone(), - transaction_manager, - template_manager, - dry_run_transaction_processor, - } - } -} - -impl JsonRpcHandlers { - pub fn rpc_discover(&self, value: JsonRpcExtractor) -> JrpcResult { - Ok(JsonRpcResponse::success( - value.id.clone(), - serde_json::from_str::>(include_str!("../../openrpc.json")).map_err(|e| { - JsonRpcResponse::error( - value.id, - JsonRpcError::new(JsonRpcErrorReason::InternalError, e.to_string(), json!({})), - ) - })?, - )) - } - - pub async fn get_identity(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let info = self - .networking - .get_local_peer_info() - .await - .map_err(internal_error(answer_id.clone()))?; - let response = GetIdentityResponse { - peer_id: info.peer_id.to_string(), - public_key: self.keypair.public_key().to_byte_type(), - public_addresses: info.listen_addrs, - }; - - Ok(JsonRpcResponse::success(answer_id, response)) - } - - pub async fn add_peer(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let AddPeerRequest { - public_key, - addresses, - wait_for_dial, - } = value.parse_params()?; - - if let Some(unsupported) = addresses.iter().find(|a| !is_supported_multiaddr(a)) { - return Err(JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::InvalidParams, - format!("Unsupported multiaddr {unsupported}"), - json::Value::Null, - ), - )); - } - - let mut networking = self.networking.clone(); - let public_key = RistrettoPublicKey::convert_from_byte_type(&public_key).map_err(|_| { - JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::InvalidParams, - "Public key is malformed".to_string(), - json::Value::Null, - ), - ) - })?; - let peer_id = public_key_to_peer_id(public_key); - - let dial_wait = networking - .dial_peer( - DialOpts::peer_id(peer_id) - .addresses(addresses) - .condition(PeerCondition::Always) - .build(), - ) - .await - .map_err(internal_error(answer_id.clone()))?; - - if wait_for_dial { - dial_wait.await.map_err(internal_error(answer_id.clone()))?; - } - - Ok(JsonRpcResponse::success(answer_id, AddPeerResponse {})) - } - - pub async fn get_comms_stats(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let peers = self - .networking - .clone() - .get_connected_peers() - .await - .map_err(internal_error(answer_id.clone()))?; - - let status = if peers.is_empty() { "Offline" } else { "Online" }; - Ok(JsonRpcResponse::success(answer_id, GetCommsStatsResponse { - connection_status: status.to_string(), - })) - } - - pub async fn get_connections(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let active_connections = self - .networking - .get_active_connections() - .await - .map_err(internal_error(answer_id.clone()))?; - - let connections = active_connections - .into_iter() - .map(|conn| types::Connection { - connection_id: conn.connection_id.to_string(), - peer_id: conn.peer_id.to_string(), - address: conn.endpoint.get_remote_address().clone(), - direction: if conn.endpoint.is_dialer() { - ConnectionDirection::Outbound - } else { - ConnectionDirection::Inbound - }, - age: conn.age(), - ping_latency: conn.ping_latency, - user_agent: conn.user_agent.map(|arc| arc.deref().clone()), - }) - .collect(); - - Ok(JsonRpcResponse::success(answer_id, GetConnectionsResponse { - connections, - })) - } - - pub async fn list_substates(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let ListSubstatesRequest { - filter_by_template, - filter_by_type, - limit, - offset, - } = value.parse_params()?; - - let substates = self - .substate_manager - .get_stored_substates_by_filters(filter_by_type, filter_by_template, limit, offset) - .map_err(|e| { - warn!(target: LOG_TARGET, "Error getting substate: {}", e); - Self::internal_error(answer_id.clone(), format!("Error getting substate: {}", e)) - })?; - - Ok(JsonRpcResponse::success(answer_id, ListSubstatesResponse { substates })) - } - - pub async fn get_substate(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let request: GetSubstateRequest = value.parse_params()?; - - let maybe_substate = self - .substate_manager - .get_substate(&request.address, request.version) - .await - .map_err(|e| { - warn!(target: LOG_TARGET, "Error getting substate: {}", e); - Self::internal_error(answer_id.clone(), format!("Error getting substate: {}", e)) - })?; - - match maybe_substate { - Some(substate_resp) => Ok(JsonRpcResponse::success(answer_id, GetSubstateResponse { - address: substate_resp.address, - version: substate_resp.version, - substate: substate_resp.substate, - })), - None => { - if request.local_search_only { - Err(JsonRpcResponse::error( - answer_id, - JsonRpcError::new( - JsonRpcErrorReason::ApplicationError(404), - format!( - "Substate {} (version:>={}) not found", - request.address, - request.version.unwrap_or(0) - ), - Value::Null, - ), - )) - } else { - // Ask network - let substate = self - .transaction_manager - .get_substate(&SubstateRequirement::new(request.address.clone(), request.version)) - .await - .map_err(|e| { - warn!(target: LOG_TARGET, "Error asking network for substate: {}", e); - JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::ApplicationError(501), - format!("Error asking network for substate:{}", e), - Value::Null, - ), - ) - })?; - match substate { - SubstateResult::DoesNotExist => Err(JsonRpcResponse::error( - answer_id, - JsonRpcError::new( - JsonRpcErrorReason::ApplicationError(404), - format!( - "Substate {} (version:>={}) not found, and not found on network", - request.address, - request.version.unwrap_or(0) - ), - Value::Null, - ), - )), - SubstateResult::Up { id, substate } => { - Ok(JsonRpcResponse::success(answer_id, GetSubstateResponse { - address: id, - version: substate.version(), - substate: substate.into_substate_value(), - })) - }, - SubstateResult::Down { version, .. } => Err(JsonRpcResponse::error( - answer_id, - JsonRpcError::new( - JsonRpcErrorReason::ApplicationError(301), - format!( - "Substate {} (version:>={}) not found, but found in a down state on network at \ - version {}", - request.address, - request.version.unwrap_or(0) + 1, - version - ), - Value::Null, - ), - )), - } - } - }, - } - } - - pub async fn get_substates(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let req: GetSubstatesRequest = value.parse_params()?; - - const MAX_REQUESTS: usize = 20; - - let GetSubstatesRequest { requests } = req; - - if requests.len() > MAX_REQUESTS { - return Err(Self::invalid_params( - answer_id, - format!("Cannot request more than {MAX_REQUESTS} substates at once"), - )); - } - - let substates = self.substate_manager.get_substates(requests.as_slice()).map_err(|e| { - warn!(target: LOG_TARGET, "Error getting substate: {}", e); - Self::internal_error(answer_id.clone(), format!("Error getting substate: {}", e)) - })?; - - Ok(JsonRpcResponse::success(answer_id, GetSubstatesResponse { substates })) - } - - pub async fn inspect_substate(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let request: InspectSubstateRequest = value.parse_params()?; - - let resp = self - .substate_manager - .get_substate(&request.address, request.version) - .await - .map_err(|e| { - warn!(target: LOG_TARGET, "Error getting substate: {}", e); - Self::internal_error(answer_id.clone(), format!("Error getting substate: {}", e)) - })? - .ok_or_else(|| { - JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::ApplicationError(404), - format!( - "Substate {} (version:>={}) not found", - request.address, - request.version.unwrap_or(0) - ), - Value::Null, - ), - ) - })?; - - Ok(JsonRpcResponse::success(answer_id, InspectSubstateResponse { - address: resp.address, - version: resp.version, - substate: resp.substate, - })) - } - - pub async fn get_non_fungibles(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let request: GetNonFungiblesRequest = value.parse_params()?; - let limit = usize::try_from(request.end_index.saturating_sub(request.start_index)).map_err(|e| { - JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::InvalidParams, - format!("Invalid end_index: {}", e), - json::Value::Null, - ), - ) - })?; - let offset = usize::try_from(request.start_index).map_err(|e| { - JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::InvalidParams, - format!("Invalid start_index: {}", e), - json::Value::Null, - ), - ) - })?; - - let non_fungibles = self - .substate_manager - .get_non_fungibles_by_resource_address(request.address, limit, offset) - .map_err(|e| Self::internal_error(answer_id.clone(), format!("Error getting non fungibles: {}", e)))?; - Ok(JsonRpcResponse::success(answer_id, GetNonFungiblesResponse { - non_fungibles, - })) - } - - pub async fn get_utxo_updates(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let req: GetUtxoUpdatesRequest = value.parse_params()?; - if req.per_shard_limit > 1000 { - return Err(JsonRpcResponse::error( - answer_id, - JsonRpcError::new( - JsonRpcErrorReason::InvalidParams, - "per_shard_limit cannot be greater than 1000".to_string(), - Value::Null, - ), - )); - } - - if req.shard_state_versions.len() > NumPreshards::MAX_SHARD.as_u32() as usize { - return Err(JsonRpcResponse::error( - answer_id, - JsonRpcError::new( - JsonRpcErrorReason::InvalidParams, - format!( - "shard_state_versions cannot contain more than {} entries", - NumPreshards::MAX_SHARD.as_u32() - ), - Value::Null, - ), - )); - } - - if req - .shard_state_versions - .keys() - .any(|shard| shard.is_global() || *shard > NumPreshards::MAX_SHARD) - { - return Err(JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::InvalidParams, - format!( - "shard_state_versions contains invalid shard. Cannot query UTXOs in global shard or greater \ - than max shard {} exceeded", - NumPreshards::MAX_SHARD.as_u32() - ), - Value::Null, - ), - )); - } - - let mut utxo_updates = HashMap::new(); - let mut per_shard_high_watermark = Vec::with_capacity(req.shard_state_versions.len()); - for (shard, state_version) in req.shard_state_versions { - let (max_state_version, updates) = self - .substate_manager - .get_utxo_updates(req.resource_address, shard, state_version, req.per_shard_limit) - .map_err(|e| { - Self::internal_error( - answer_id.clone(), - format!( - "Error getting UTXO updates for resource_address {}, shard {}, state_version {}: {}", - req.resource_address, shard, state_version, e - ), - ) - })?; - - // TODO: this is on the hot path, figure out a better way to let the client know the max shard versions - // without sending it each time - let current_tip_version = self - .substate_manager - .get_max_state_version(&req.resource_address, shard) - .map_err(|e| { - Self::internal_error( - answer_id.clone(), - format!( - "Error getting max state version for resource_address {}, shard {}: {}", - req.resource_address, shard, e - ), - ) - })?; - if current_tip_version.as_u64() > 0 { - // Save a little over the wire initially by not sending 0 watermarks - per_shard_high_watermark.push((shard, current_tip_version)); - } - if !updates.is_empty() { - utxo_updates.insert(shard, UtxoStateUpdateSet { - updates, - max_state_version, - }); - } - } - - Ok(JsonRpcResponse::success(answer_id, GetUtxoUpdatesResponse { - updates: UtxoUpdateSet { - shard_updates: utxo_updates, - per_shard_high_watermark, - }, - })) - } - - pub async fn get_unspent_utxos(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let req: GetUnspentUtxosRequest = value.parse_params()?; - if req.tag_and_nonce_pairs.len() > 1000 { - return Err(JsonRpcResponse::error( - answer_id, - JsonRpcError::new( - JsonRpcErrorReason::InvalidParams, - "cannot query more than 1000 UTXOs".to_string(), - Value::Null, - ), - )); - } - let utxos = self - .substate_manager - .get_unspent_utxos(&req.resource_address, &req.tag_and_nonce_pairs) - .map_err(|e| { - Self::internal_error( - answer_id.clone(), - format!( - "Error getting UTXOs for resource_address {}, with {} tag/nonce pair(s): {}", - req.resource_address, - req.tag_and_nonce_pairs.len(), - e - ), - ) - })?; - - Ok(JsonRpcResponse::success(answer_id, GetUnspentUtxosResponse { utxos })) - } - - pub async fn submit_transaction(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let request: SubmitTransactionRequest = value.parse_params()?; - - if request.is_dry_run { - let transaction_id = request.transaction.calculate_id(); - let exec_result = self - .dry_run_transaction_processor - .process_transaction(request.transaction) - .await - .map_err(|e| Self::internal_error(answer_id.clone(), e))?; - - return Ok(JsonRpcResponse::success(answer_id, SubmitTransactionResponse { - result: IndexerTransactionFinalizedResult::Finalized { - execution_result: Some(Box::new(exec_result)), - final_decision: Decision::Commit, - abort_details: None, - finalized_time: now(), - execution_time: Default::default(), - }, - transaction_id, - })); - } - - let transaction = request.transaction; - let transaction_id = self - .transaction_manager - .submit_transaction(transaction) - .await - .map_err(|e| match e { - TransactionManagerError::NetworkClientError(NetworkClientError::AllValidatorsFailed { .. }) => { - JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::ApplicationError(400), - format!("All validators failed: {}", e), - json::Value::Null, - ), - ) - }, - TransactionManagerError::InvalidTransaction { - transaction_id, - details, - } => JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::ApplicationError(400), - format!("Transaction {} is invalid: {}", transaction_id, details), - json::Value::Null, - ), - ), - e => Self::internal_error(answer_id.clone(), e), - })?; - - info!(target: LOG_TARGET, "✅ Transaction submitted: {}", transaction_id); - - Ok(JsonRpcResponse::success(answer_id, SubmitTransactionResponse { - result: IndexerTransactionFinalizedResult::Pending, - transaction_id, - })) - } - - pub async fn get_epoch_manager_stats(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let current_epoch = self.epoch_manager.get_current_epoch(); - let current_epoch_hash = self - .epoch_manager - .get_current_epoch_hash() - .await - .map_err(internal_error(answer_id.clone()))?; - - let current_block_height = self - .global_db - .create_transaction() - .and_then(|mut tx| { - self.global_db - .metadata(&mut tx) - .get_metadata::(StoreKey::BaseLayerLastScannedBlockHeight.as_key_bytes()) - }) - .map_err(|e| { - JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::InternalError, - format!("Could not get current block height: {}", e), - json::Value::Null, - ), - ) - })?; - - let response = GetEpochManagerStatsResponse { - current_epoch, - current_block_height: current_block_height.unwrap_or(0), - current_block_hash: current_epoch_hash, - }; - Ok(JsonRpcResponse::success(answer_id, response)) - } - - pub async fn get_network_sync_state(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let network_desc = self - .epoch_manager - .get_network_description() - .await - .map_err(internal_error(answer_id.clone()))?; - let sync_progress = self - .substate_manager - .get_sync_progress() - .optional() - .map_err(internal_error(answer_id.clone()))?; - - let response = GetNetworkSyncStateResponse { - network_desc: NetworkDescription { - epoch: network_desc.epoch, - shard_groups: network_desc - .shard_groups - .into_iter() - .map(|(shard_group, info)| (shard_group, info.num_members)) - .collect(), - num_preshards: network_desc.num_preshards, - }, - sync_progress: sync_progress.map(|p| SyncProgress { - last_epoch: p.last_epoch, - checkpoint_progress: p.checkpoint_progress.into_iter().collect(), - last_state_versions: p.last_state_versions.into_iter().collect(), - }), - }; - Ok(JsonRpcResponse::success(answer_id, response)) - } - - pub async fn get_template_definition(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let request: GetTemplateDefinitionRequest = value.parse_params()?; - - let template = self - .template_manager - .fetch_template(&request.template_address) - .optional() - .map_err(|err| { - // If it's pending, we return a 404 to the client - this allows them to retry later - if matches!(err, TemplateManagerError::TemplateUnavailable { - status: Some(TemplateStatus::Pending) - }) { - Self::not_found( - answer_id.clone(), - format!( - "Template with address {} is still being downloaded. Try again later.", - request.template_address - ), - ) - } else { - Self::internal_error(answer_id.clone(), format!("Error fetching template: {}", err)) - } - })? - .ok_or_else(|| { - Self::not_found( - answer_id.clone(), - format!("Template with address {} not found", request.template_address), - ) - })?; - let template = match template.executable { - TemplateExecutable::CompiledWasm(code) => WasmModule::from_code(code) - .load_template() - .map_err(|e| Self::internal_error(answer_id.clone(), format!("Error loading template: {}", e)))?, - // TemplateExecutable::DownloadableWasm is never returned ad there is no DB type for that - TemplateExecutable::DownloadableWasm(_, _) | TemplateExecutable::Manifest(_) => { - return Err(JsonRpcResponse::error( - answer_id, - JsonRpcError::new( - JsonRpcErrorReason::InvalidRequest, - "Template is not a wasm module".to_string(), - json::Value::Null, - ), - )); - }, - }; - - Ok(JsonRpcResponse::success(answer_id, GetTemplateDefinitionResponse { - definition: template.template_def().clone(), - name: template.template_name().to_string(), - })) - } - - pub async fn list_templates(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let req: ListTemplatesRequest = value.parse_params()?; - let templates = self - .template_manager - .fetch_template_metadata(req.limit as usize) - .map_err(|e| Self::internal_error(answer_id.clone(), e))?; - - Ok(JsonRpcResponse::success(answer_id, ListTemplatesResponse { - templates: templates - .into_iter() - .map(|t| TemplateMetadata { - name: t.name, - address: t.address, - binary_sha: to_hex(t.binary_sha.as_slice()), - }) - .collect(), - })) - } - - pub async fn get_transaction_result(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let request: GetTransactionResultRequest = value.parse_params()?; - - let result = self - .transaction_manager - .get_transaction_result(request.transaction_id) - .await - .optional() - .map_err(|e| Self::internal_error(answer_id.clone(), e))? - .ok_or_else(|| Self::not_found(answer_id.clone(), "Transaction not found"))?; - - let resp = match result { - TransactionResultStatus::Pending => GetTransactionResultResponse { - result: IndexerTransactionFinalizedResult::Pending, - }, - TransactionResultStatus::Finalized(finalized) => GetTransactionResultResponse { - result: IndexerTransactionFinalizedResult::Finalized { - final_decision: finalized.final_decision, - execution_result: finalized.execute_result.map(Box::new), - execution_time: finalized.execution_time, - finalized_time: finalized.finalized_time, - abort_details: finalized.abort_details, - }, - }, - }; - - Ok(JsonRpcResponse::success(answer_id, resp)) - } - - pub async fn list_recent_transactions(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - let req: ListRecentTransactionsRequest = value.parse_params()?; - if req.limit.is_some_and(|l| l > 1000) { - return Err(JsonRpcResponse::error( - answer_id, - JsonRpcError::new( - JsonRpcErrorReason::InvalidParams, - "Limit cannot be greater than 1000".to_string(), - json::Value::Null, - ), - )); - } - - let limit = req.limit.unwrap_or(100); - if limit > 1000 { - return Err(JsonRpcResponse::error( - answer_id.clone(), - JsonRpcError::new( - JsonRpcErrorReason::InvalidParams, - "Limit cannot be greater than 1000".to_string(), - json::Value::Null, - ), - )); - } - - let transactions = self - .transaction_manager - .list_recent_transactions(None, limit as usize) - .map_err(|e| Self::internal_error(answer_id.clone(), e))?; - - let resp = ListRecentTransactionsResponse { transactions }; - Ok(JsonRpcResponse::success(answer_id, resp)) - } - - pub async fn wait_until_ready(&self, value: JsonRpcExtractor) -> JrpcResult { - let answer_id = value.get_answer_id(); - // TODO: we should rather return the current state of the indexer and have clients poll - self.epoch_manager - .wait_for_initial_scanning_to_complete() - .await - .map_err(internal_error(answer_id.clone()))?; - - Ok(JsonRpcResponse::success(answer_id, IndexerReadyResponse {})) - } - - fn error_response(answer_id: axum_jrpc::Id, reason: JsonRpcErrorReason, message: T) -> JsonRpcResponse { - JsonRpcResponse::error( - answer_id, - JsonRpcError::new(reason, message.to_string(), json::Value::Null), - ) - } - - fn not_found(answer_id: axum_jrpc::Id, details: T) -> JsonRpcResponse { - Self::error_response(answer_id, JsonRpcErrorReason::ApplicationError(404), details) - } - - fn invalid_params(answer_id: axum_jrpc::Id, details: T) -> JsonRpcResponse { - Self::error_response(answer_id, JsonRpcErrorReason::InvalidParams, details) - } - - fn internal_error(answer_id: axum_jrpc::Id, error: T) -> JsonRpcResponse { - error!(target: LOG_TARGET, "Internal error: {}", error); - Self::error_response(answer_id, JsonRpcErrorReason::InternalError, error) - } -} - -fn now() -> PrimitiveDateTime { - let now = UtcDateTime::now(); - PrimitiveDateTime::new(now.date(), now.time()) -} diff --git a/applications/tari_indexer/src/json_rpc/mod.rs b/applications/tari_indexer/src/json_rpc/mod.rs deleted file mode 100644 index 5e06fb8e9d..0000000000 --- a/applications/tari_indexer/src/json_rpc/mod.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2023. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -mod handlers; -pub use handlers::JsonRpcHandlers; - -mod error; -// mod json_encoding; -mod server; - -pub use server::spawn_json_rpc; diff --git a/applications/tari_indexer/src/json_rpc/server.rs b/applications/tari_indexer/src/json_rpc/server.rs deleted file mode 100644 index ae0f44858e..0000000000 --- a/applications/tari_indexer/src/json_rpc/server.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2023. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use std::{future::IntoFuture, net::SocketAddr, sync::Arc}; - -use axum::{ - extract::{DefaultBodyLimit, Extension}, - routing::post, - Router, -}; -use axum_jrpc::{JrpcResult, JsonRpcExtractor}; -use log::*; -use tari_ootle_app_utilities::tcp::try_bind_with_fallback; -use tower_http::cors::CorsLayer; - -use super::handlers::JsonRpcHandlers; - -const LOG_TARGET: &str = "tari::indexer::json_rpc"; - -pub async fn spawn_json_rpc(preferred_address: SocketAddr, handlers: JsonRpcHandlers) -> anyhow::Result { - let router = Router::new() - .route("/", post(handler)) - .route("/json_rpc", post(handler)) - .layer(Extension(Arc::new(handlers))) - // Limit the body size to 5MB to allow for large transactions (wasm uploads) - .layer(DefaultBodyLimit::max(5*1024*1024)) - .layer(CorsLayer::permissive()); - - let listener = try_bind_with_fallback(preferred_address).await?; - let server = axum::serve(listener, router); - let listen_addr = server.local_addr()?; - info!(target: LOG_TARGET, "🌐 JSON-RPC listening on {listen_addr}"); - tokio::spawn(server.into_future()); - - Ok(listen_addr) -} - -async fn handler(Extension(handlers): Extension>, value: JsonRpcExtractor) -> JrpcResult { - info!(target: LOG_TARGET, "🌐 JSON-RPC request: {}", value.method); - debug!(target: LOG_TARGET, "🌐 JSON-RPC body: {:?}", value); - match value.method.as_str() { - "rpc.discover" => handlers.rpc_discover(value), - // P2p Network - "get_identity" => handlers.get_identity(value).await, - "add_peer" => handlers.add_peer(value).await, - "get_comms_stats" => handlers.get_comms_stats(value).await, - "get_connections" => handlers.get_connections(value).await, - - // Substates - "list_substates" => handlers.list_substates(value).await, - "get_substate" => handlers.get_substate(value).await, - "get_substates" => handlers.get_substates(value).await, - "inspect_substate" => handlers.inspect_substate(value).await, - "get_non_fungibles" => handlers.get_non_fungibles(value).await, - "get_utxo_updates" => handlers.get_utxo_updates(value).await, - "get_unspent_utxos" => handlers.get_unspent_utxos(value).await, - - // Transactions - "submit_transaction" => handlers.submit_transaction(value).await, - "get_transaction_result" => handlers.get_transaction_result(value).await, - - // Templates - "get_template_definition" => handlers.get_template_definition(value).await, - "list_templates" => handlers.list_templates(value).await, - "list_recent_transactions" => handlers.list_recent_transactions(value).await, - - // Misc - "wait_until_ready" => handlers.wait_until_ready(value).await, - "get_epoch_manager_stats" => handlers.get_epoch_manager_stats(value).await, - "get_network_sync_state" => handlers.get_network_sync_state(value).await, - method => Ok(value.method_not_found(method)), - } -} diff --git a/applications/tari_indexer/src/lib.rs b/applications/tari_indexer/src/lib.rs index fa2be616eb..ba4febfcf0 100644 --- a/applications/tari_indexer/src/lib.rs +++ b/applications/tari_indexer/src/lib.rs @@ -32,11 +32,11 @@ mod dry_run; pub mod graphql; #[cfg(feature = "web_ui")] mod http_ui; +mod rest_api; mod block_data; mod event_data; mod event_manager; -mod json_rpc; mod network_client; mod network_state_sync; mod storage_sqlite; @@ -49,23 +49,16 @@ use std::{convert::Infallible, fs, future, future::Future}; use log::*; use network_state_sync::BlockScanner; use serde::Serialize; -use substate_manager::SubstateManager; use tari_common::exit_codes::{ExitCode, ExitError}; use tari_consensus::consensus_constants::ConsensusConstants; -use tari_engine::transaction::TransactionProcessorConfig; use tari_epoch_manager::{ traits::{EpochManagerSpec, LayerOneTransactionSubmitter}, EpochManagerEvent, EpochManagerReader, }; use tari_epoch_oracles::EpochOracle; -use tari_indexer_lib::substate_scanner::SubstateScanner; use tari_networking::NetworkingService; -use tari_ootle_app_utilities::{ - claim_burn_proof_verifier::TariClaimBurnProofVerifier, - keypair::setup_keypair_prompt, - template_download_queue::TemplateDownloadQueue, -}; +use tari_ootle_app_utilities::{keypair::setup_keypair_prompt, template_download_queue::TemplateDownloadQueue}; use tari_ootle_common_types::{layer_one_transaction::LayerOneTransactionDef, PeerAddress}; use tari_ootle_storage::global::{DbFactory, GlobalDb}; use tari_ootle_storage_sqlite::{global::SqliteGlobalDbAdapter, SqliteDbFactory}; @@ -75,12 +68,8 @@ use tokio::{task, time}; use crate::{ bootstrap::{spawn_services, Services}, config::ApplicationConfig, - dry_run::processor::DryRunTransactionProcessor, event_manager::EventManager, graphql::server::run_graphql, - json_rpc::{spawn_json_rpc, JsonRpcHandlers}, - substate_file_cache::SubstateFileCache, - transaction_manager::TransactionManager, }; const LOG_TARGET: &str = "tari::indexer::app"; @@ -110,29 +99,6 @@ pub async fn run_indexer(config: ApplicationConfig, mut shutdown_signal: Shutdow let mut epoch_manager_events = services.epoch_manager.subscribe(); - let substate_cache_dir = config.common.base_path.join("substate_cache"); - let substate_cache = SubstateFileCache::new(substate_cache_dir) - .map_err(|e| ExitError::new(ExitCode::ConfigError, format!("Substate cache error: {}", e)))?; - - let substate_scanner = SubstateScanner::new( - services.epoch_manager.clone(), - services.validator_node_client_factory.clone(), - substate_cache, - ); - - let substate_manager = SubstateManager::new(substate_scanner.clone(), services.store.clone()); - let transaction_manager = TransactionManager::new(services.network_client.clone(), services.store.clone()); - - // dry run - let dry_run_transaction_processor = DryRunTransactionProcessor::new( - TransactionProcessorConfig::new(config.network) - .with_template_binary_max_size_bytes(consensus_constants.template_binary_max_size_bytes), - services.epoch_manager.clone(), - services.validator_node_client_factory.clone(), - services.template_manager.clone(), - TariClaimBurnProofVerifier::new(config.network, services.global_db.clone()), - ); - // Run the event manager let event_manager = EventManager::new(services.store.clone()); @@ -140,35 +106,27 @@ pub async fn run_indexer(config: ApplicationConfig, mut shutdown_signal: Shutdow let graphql_address = config.indexer.graphql_address; if let Some(address) = graphql_address { info!(target: LOG_TARGET, "🌐 Started GraphQL server on {}", address); - task::spawn(run_graphql(address, substate_manager.clone(), event_manager)); + task::spawn(run_graphql(address, services.substate_manager.clone(), event_manager)); } - // Run the JSON-RPC API - let jrpc_address = config.indexer.json_rpc_address; - if let Some(jrpc_address) = jrpc_address { - info!(target: LOG_TARGET, "🌐 Started JSON-RPC server on {}", jrpc_address); - let handlers = JsonRpcHandlers::new( - &services, - substate_manager, - transaction_manager, - services.global_db.clone(), - services.template_manager.clone(), - dry_run_transaction_processor, - ); - let jrpc_address = spawn_json_rpc(jrpc_address, handlers).await?; - debug!(target: LOG_TARGET, "JSON-RPC address {}", jrpc_address); + // Run the REST API + let listen_addr = config.indexer.api_listen_address; + if let Some(listen_addr) = listen_addr { + let listen_address = rest_api::Server::spawn(listen_addr, &services, shutdown_signal.clone()) + .await + .map_err(|e| ExitError::new(ExitCode::ConfigError, e))?; + debug!(target: LOG_TARGET, "API address {}", listen_address); // Run the web ui #[cfg(feature = "web_ui")] if let Some(address) = config.indexer.web_ui_address { - // json rpc - let public_jrpc_url = config + let public_api_url = config .indexer - .web_ui_public_json_rpc_url - .unwrap_or_else(|| format!("http://{jrpc_address}/json_rpc")); - let public_jrpc_address = url::Url::parse(&public_jrpc_url).map_err(|err| { + .web_ui_public_api_url + .unwrap_or_else(|| format!("http://{listen_address}")); + let public_api_address = url::Url::parse(&public_api_url).map_err(|err| { ExitError::new( ExitCode::ConfigError, - format!("Invalid public JSON-rpc url '{public_jrpc_url}': {err}"), + format!("Invalid public API url '{public_api_url}': {err}"), ) })?; @@ -190,7 +148,7 @@ pub async fn run_indexer(config: ApplicationConfig, mut shutdown_signal: Shutdow tokio::spawn(http_ui::server::run_http_ui_server( address, - public_jrpc_address, + public_api_address, public_graphql_url, )); } diff --git a/applications/tari_indexer/src/rest_api/cache.rs b/applications/tari_indexer/src/rest_api/cache.rs new file mode 100644 index 0000000000..37c775aff8 --- /dev/null +++ b/applications/tari_indexer/src/rest_api/cache.rs @@ -0,0 +1,54 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use axum::http::{header, HeaderMap, HeaderValue}; + +pub struct HttpCacheConfig { + pub is_public: bool, + pub max_age: u32, + pub s_maxage: u32, + pub stale_while_revalidate: u32, +} +impl HttpCacheConfig { + pub fn new() -> Self { + Self::default() + } + + pub fn to_header_value(&self) -> HeaderValue { + self.to_header_string() + .parse() + .expect("BUG: failed to parse cache control header value") + } + + pub fn to_header_string(&self) -> String { + format!( + "{}, max-age={}, s-maxage={}, stale-while-revalidate={}", + if self.is_public { "public" } else { "private" }, + self.max_age, + self.s_maxage, + self.stale_while_revalidate + ) + } + + pub fn apply(&self, headers: &mut HeaderMap) { + headers.insert(header::CACHE_CONTROL, self.to_header_value()); + } + + pub fn with_max_age(mut self, max_age: u32) -> Self { + self.max_age = max_age; + self.s_maxage = (max_age / 2).max(1); + self.stale_while_revalidate = (max_age / 4).max(1); + self + } +} + +impl Default for HttpCacheConfig { + fn default() -> Self { + Self { + is_public: true, + max_age: 60, + s_maxage: 30, + stale_while_revalidate: 15, + } + } +} diff --git a/applications/tari_indexer/src/rest_api/context.rs b/applications/tari_indexer/src/rest_api/context.rs new file mode 100644 index 0000000000..e5ec8546d9 --- /dev/null +++ b/applications/tari_indexer/src/rest_api/context.rs @@ -0,0 +1,116 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::sync::Arc; + +use axum::{ + http, + response::{IntoResponse, Response}, +}; +use tari_engine_types::ToByteType; +use tari_epoch_manager::service::EpochManagerHandle; +use tari_networking::NetworkingHandle; +use tari_ootle_common_types::PeerAddress; +use tari_ootle_p2p::TariMessagingSpec; +use tari_ootle_storage::global::GlobalDb; +use tari_ootle_storage_sqlite::global::SqliteGlobalDbAdapter; +use tari_template_lib::types::crypto::RistrettoPublicKeyBytes; +use tari_template_manager::implementation::TemplateManager; +use tari_validator_node_rpc::client::TariValidatorNodeRpcClientFactory; + +use crate::{ + bootstrap::Services, + dry_run::processor::DryRunTransactionProcessor, + rest_api::cache::HttpCacheConfig, + storage_sqlite::SqliteIndexerStore, + substate_manager::SubstateManager, + transaction_manager::TransactionManager, +}; + +#[derive(Clone)] +pub struct HandlerContext { + inner: Arc, +} + +impl HandlerContext { + pub fn from_services(services: &Services) -> Self { + Self { + inner: Arc::new(InnerContext { + cache_control_enabled: true, + global_db: services.global_db.clone(), + epoch_manager: services.epoch_manager.clone(), + networking: services.networking.clone(), + public_key: services.keypair.public_key().to_byte_type(), + substate_manager: services.substate_manager.clone(), + transaction_manager: services.transaction_manager.clone(), + template_manager: services.template_manager.clone(), + dry_run_transaction_processor: services.dry_run_transaction_processor.clone(), + }), + } + } + + pub fn global_db(&self) -> &GlobalDb> { + &self.inner.global_db + } + + pub fn epoch_manager(&self) -> &EpochManagerHandle { + &self.inner.epoch_manager + } + + pub fn networking(&self) -> &NetworkingHandle { + &self.inner.networking + } + + pub fn public_key(&self) -> &RistrettoPublicKeyBytes { + &self.inner.public_key + } + + pub fn is_cache_control_enabled(&self) -> bool { + self.inner.cache_control_enabled + } + + pub fn substate_manager(&self) -> &SubstateManager { + &self.inner.substate_manager + } + + pub fn transaction_manager( + &self, + ) -> &TransactionManager, TariValidatorNodeRpcClientFactory, SqliteIndexerStore> + { + &self.inner.transaction_manager + } + + pub fn template_manager(&self) -> &TemplateManager { + &self.inner.template_manager + } + + pub fn dry_run_transaction_processor(&self) -> &DryRunTransactionProcessor { + &self.inner.dry_run_transaction_processor + } + + pub fn apply_cache_control(&self, body: impl IntoResponse, max_age: u32) -> Response { + let mut response = body.into_response(); + let headers = response.headers_mut(); + self.apply_custom_cache_control(headers, &HttpCacheConfig::new().with_max_age(max_age)); + response + } + + pub fn apply_custom_cache_control(&self, headers: &mut http::HeaderMap, config: &HttpCacheConfig) { + if self.is_cache_control_enabled() { + config.apply(headers); + } + } +} + +struct InnerContext { + cache_control_enabled: bool, + global_db: GlobalDb>, + epoch_manager: EpochManagerHandle, + networking: NetworkingHandle, + public_key: RistrettoPublicKeyBytes, + substate_manager: SubstateManager, + transaction_manager: + TransactionManager, TariValidatorNodeRpcClientFactory, SqliteIndexerStore>, + template_manager: TemplateManager, + dry_run_transaction_processor: DryRunTransactionProcessor, +} diff --git a/applications/tari_indexer/src/rest_api/encoder.rs b/applications/tari_indexer/src/rest_api/encoder.rs new file mode 100644 index 0000000000..4431dc683e --- /dev/null +++ b/applications/tari_indexer/src/rest_api/encoder.rs @@ -0,0 +1,9 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use bytes::BufMut; + +pub trait Encoder { + type Item; + fn encode_into(&self, msg: &Self::Item, buf: &mut impl BufMut) -> anyhow::Result<()>; +} diff --git a/applications/tari_indexer/src/rest_api/error.rs b/applications/tari_indexer/src/rest_api/error.rs new file mode 100644 index 0000000000..4c7b09783d --- /dev/null +++ b/applications/tari_indexer/src/rest_api/error.rs @@ -0,0 +1,112 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::borrow::Cow; + +use axum::{http::StatusCode, response::IntoResponse, Json}; +use log::*; +use serde_json::json; +use utoipa::{ + openapi::{RefOr, Schema}, + PartialSchema, + ToSchema, +}; + +const LOG_TARGET: &str = "tari::indexer::rest_api::error"; + +#[macro_export] +macro_rules! bailout { + ($msg:literal $(,)?) => { + { + let error = $crate::rest_api::error::ErrorResponse::anyhow(anyhow::format_err!($msg)); + return Err(error); + } + }; + ($fmt:expr, $($arg:tt)*) => { + return Err($crate::rest_api::error::ErrorResponse::anyhow(anyhow::Error::msg(format!($fmt, $($arg)*)))); + }; +} + +#[derive(Debug, Clone)] +pub struct ErrorResponse { + pub status: StatusCode, + pub error: Box, +} + +#[derive(ToSchema)] +struct ErrorResp { + #[allow(unused)] + pub error: Box, +} + +impl ErrorResponse { + #[must_use] + pub fn anyhow>(err: E) -> Self { + let err = err.into(); + error!(target: LOG_TARGET, "Internal server error: {}", err); + Self::internal_error(err.to_string()) + } + + #[must_use] + pub fn internal_error(msg: impl Into>) -> Self { + let msg = msg.into(); + error!(target: LOG_TARGET, "Internal server error: {}", msg); + let msg = if cfg!(debug_assertions) || option_env!("API_DEBUG").is_some_and(|v| v != "0") { + msg + } else { + "Internal server error".into() + }; + Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + error: msg, + } + } + + #[must_use] + pub fn not_found(msg: impl Into>) -> Self { + Self { + status: StatusCode::NOT_FOUND, + error: msg.into(), + } + } + + #[must_use] + pub fn bad_request(msg: impl Into>) -> Self { + Self { + status: StatusCode::BAD_REQUEST, + error: msg.into(), + } + } + + #[must_use] + pub fn service_unavailable(msg: impl Into>) -> Self { + Self { + status: StatusCode::SERVICE_UNAVAILABLE, + error: msg.into(), + } + } +} + +impl From for ErrorResponse { + fn from(err: anyhow::Error) -> Self { + Self::anyhow(err) + } +} + +impl IntoResponse for ErrorResponse { + fn into_response(self) -> axum::response::Response { + (self.status, Json(json!({"error": self.error}))).into_response() + } +} + +impl PartialSchema for ErrorResponse { + fn schema() -> RefOr { + ErrorResp::schema() + } +} + +impl ToSchema for ErrorResponse { + fn name() -> Cow<'static, str> { + Cow::Borrowed("ErrorResponse") + } +} diff --git a/applications/tari_indexer/src/rest_api/handlers/misc.rs b/applications/tari_indexer/src/rest_api/handlers/misc.rs new file mode 100644 index 0000000000..9b9d0c6b56 --- /dev/null +++ b/applications/tari_indexer/src/rest_api/handlers/misc.rs @@ -0,0 +1,84 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use axum::{response::Response, Extension, Json}; +use tari_epoch_manager::EpochManagerReader; +use tari_epoch_oracles::store::StoreKey; +use tari_indexer_client::{ + rest_api::types::GetIdentityResponse, + types::{GetEpochManagerStatsResponse, IndexerReadyResponse}, +}; + +use crate::rest_api::{context::HandlerContext, error::ErrorResponse, handlers::HandlerResult}; + +#[utoipa::path( + get, + path = "/identity", + description = "Get indexer network identity information", + responses( + (status = 200, description = "Indexer network identity info", body = GetIdentityResponse), + (status = INTERNAL_SERVER_ERROR, description = "Failed to get identity info", body = ErrorResponse), + ), +)] +pub async fn get_identity(Extension(context): Extension) -> HandlerResult { + let info = context + .networking() + .get_local_peer_info() + .await + .map_err(ErrorResponse::anyhow)?; + let response = GetIdentityResponse { + peer_id: info.peer_id.to_string(), + public_key: *context.public_key(), + public_addresses: info.listen_addrs, + }; + Ok(context.apply_cache_control(Json(response), 1000)) +} + +#[utoipa::path( + get, + path = "/wait-until-ready", + description = "Request will return an empty result when the indexer is ready to serve requests. This may take an \ + indefinite amount of time after startup." +)] +pub async fn wait_until_ready( + Extension(context): Extension, +) -> HandlerResult> { + // TODO: we should rather return the current state of the indexer and have clients poll + context + .epoch_manager() + .wait_for_initial_scanning_to_complete() + .await + .map_err(ErrorResponse::anyhow)?; + + Ok(Json(IndexerReadyResponse {})) +} + +#[utoipa::path(get, path = "/epoch-manager/stats", description = "Get epoch manager stats")] +pub async fn get_epoch_manager_stats( + Extension(context): Extension, +) -> HandlerResult> { + let current_epoch = context.epoch_manager().get_current_epoch(); + let current_epoch_hash = context + .epoch_manager() + .get_current_epoch_hash() + .await + .map_err(ErrorResponse::anyhow)?; + + let current_block_height = context + .global_db() + .create_transaction() + .and_then(|mut tx| { + context + .global_db() + .metadata(&mut tx) + .get_metadata::(StoreKey::BaseLayerLastScannedBlockHeight.as_key_bytes()) + }) + .map_err(ErrorResponse::anyhow)?; + + let response = GetEpochManagerStatsResponse { + current_epoch, + current_block_height: current_block_height.unwrap_or(0), + current_block_hash: current_epoch_hash, + }; + Ok(Json(response)) +} diff --git a/applications/tari_indexer/src/rest_api/handlers/mod.rs b/applications/tari_indexer/src/rest_api/handlers/mod.rs new file mode 100644 index 0000000000..21a22a345d --- /dev/null +++ b/applications/tari_indexer/src/rest_api/handlers/mod.rs @@ -0,0 +1,12 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +pub mod misc; +pub mod network; +pub mod nfts; +pub mod substates; +pub mod templates; +pub mod transactions; +pub mod utxos; + +pub type HandlerResult = Result; diff --git a/applications/tari_indexer/src/rest_api/handlers/network.rs b/applications/tari_indexer/src/rest_api/handlers/network.rs new file mode 100644 index 0000000000..3f9f122a2e --- /dev/null +++ b/applications/tari_indexer/src/rest_api/handlers/network.rs @@ -0,0 +1,126 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::ops::Deref; + +use axum::{response::Response, Extension, Json}; +use libp2p::swarm::dial_opts::PeerCondition; +use tari_engine_types::FromByteType; +use tari_indexer_client::{ + types, + types::{ + AddPeerRequest, + AddPeerResponse, + ConnectionDirection, + GetConnectionsResponse, + GetNetworkSyncStateResponse, + NetworkDescription, + SyncProgress, + }, +}; +use tari_networking::{is_supported_multiaddr, DialOpts, NetworkingService}; +use tari_ootle_common_types::{optional::Optional, public_key_to_peer_id}; + +use crate::rest_api::{context::HandlerContext, error::ErrorResponse, handlers::HandlerResult}; + +#[utoipa::path(get, path = "/network/stats", description = "Get network sync stats")] +pub async fn get_network_sync_stats( + Extension(context): Extension, +) -> HandlerResult> { + let network_desc = context + .epoch_manager() + .get_network_description() + .await + .map_err(ErrorResponse::anyhow)?; + let sync_progress = context + .substate_manager() + .get_sync_progress() + .optional() + .map_err(ErrorResponse::anyhow)?; + + let response = GetNetworkSyncStateResponse { + network_desc: NetworkDescription { + epoch: network_desc.epoch, + shard_groups: network_desc + .shard_groups + .into_iter() + .map(|(shard_group, info)| (shard_group, info.num_members)) + .collect(), + num_preshards: network_desc.num_preshards, + }, + sync_progress: sync_progress.map(|p| SyncProgress { + last_epoch: p.last_epoch, + checkpoint_progress: p.checkpoint_progress.into_iter().collect(), + last_state_versions: p.last_state_versions.into_iter().collect(), + }), + }; + Ok(Json(response)) +} + +#[utoipa::path(get, path = "/network/connections", description = "Get active peer connections")] +pub async fn get_connections(Extension(context): Extension) -> HandlerResult { + let active_connections = context + .networking() + .get_active_connections() + .await + .map_err(ErrorResponse::anyhow)?; + + let connections = active_connections + .into_iter() + .map(|conn| types::Connection { + connection_id: conn.connection_id.to_string(), + peer_id: conn.peer_id.to_string(), + address: conn.endpoint.get_remote_address().clone(), + direction: if conn.endpoint.is_dialer() { + ConnectionDirection::Outbound + } else { + ConnectionDirection::Inbound + }, + age: conn.age(), + ping_latency: conn.ping_latency, + user_agent: conn.user_agent.map(|arc| arc.deref().clone()), + }) + .collect(); + + Ok(context.apply_cache_control(Json(GetConnectionsResponse { connections }), 10)) +} + +#[utoipa::path(post, path = "/network/connections", description = "Add a new peer connection")] +pub async fn add_connection( + Extension(context): Extension, + Json(req): Json, +) -> HandlerResult> { + let AddPeerRequest { + public_key, + addresses, + wait_for_dial, + } = req; + + if let Some(unsupported) = addresses.iter().find(|a| !is_supported_multiaddr(a)) { + return Err(ErrorResponse::bad_request(format!( + "Unsupported multiaddr {unsupported}" + ))); + } + + let mut networking = context.networking().clone(); + let public_key = public_key + .try_from_byte_type() + .map_err(|_| ErrorResponse::bad_request("Public key is malformed"))?; + let peer_id = public_key_to_peer_id(public_key); + + let dial_wait = networking + .dial_peer( + DialOpts::peer_id(peer_id) + .addresses(addresses) + .condition(PeerCondition::Always) + .build(), + ) + .await + .map_err(ErrorResponse::anyhow)?; + + if wait_for_dial { + dial_wait.await.map_err(ErrorResponse::anyhow)?; + } + + Ok(Json(AddPeerResponse {})) +} diff --git a/applications/tari_indexer/src/rest_api/handlers/nfts.rs b/applications/tari_indexer/src/rest_api/handlers/nfts.rs new file mode 100644 index 0000000000..be4838b6bc --- /dev/null +++ b/applications/tari_indexer/src/rest_api/handlers/nfts.rs @@ -0,0 +1,36 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use axum::{extract::Query, Extension, Json}; +use tari_indexer_client::types::{GetNonFungiblesRequest, GetNonFungiblesResponse}; + +use crate::rest_api::{context::HandlerContext, error::ErrorResponse, handlers::HandlerResult}; + +#[utoipa::path(post, path = "/non-fungibles", description = "Get non-fungibles by resource address")] +pub async fn get_non_fungibles( + Extension(context): Extension, + Query(req): Query, +) -> HandlerResult> { + if req.end_index < req.start_index { + return Err(ErrorResponse::bad_request(format!( + "end_index ({}) must be greater than or equal to start_index ({})", + req.end_index, req.start_index + ))); + } + let limit = usize::try_from(req.end_index.saturating_sub(req.start_index)) + .map_err(|e| ErrorResponse::bad_request(format!("Invalid end_index: {}", e)))?; + let offset = usize::try_from(req.start_index) + .map_err(|e| ErrorResponse::bad_request(format!("Invalid start_index: {}", e)))?; + if limit > 1000 { + return Err(ErrorResponse::bad_request( + "The maximum allowed limit is 1000".to_string(), + )); + } + + let non_fungibles = context + .substate_manager() + .get_non_fungibles_by_resource_address(req.address, limit, offset) + .map_err(ErrorResponse::anyhow)?; + + Ok(Json(GetNonFungiblesResponse { non_fungibles })) +} diff --git a/applications/tari_indexer/src/rest_api/handlers/substates.rs b/applications/tari_indexer/src/rest_api/handlers/substates.rs new file mode 100644 index 0000000000..c743193211 --- /dev/null +++ b/applications/tari_indexer/src/rest_api/handlers/substates.rs @@ -0,0 +1,135 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use anyhow::anyhow; +use axum::{ + extract::{Path, Query}, + response::Response, + Extension, + Json, +}; +use tari_engine_types::substate::SubstateId; +use tari_indexer_client::types::{ + GetSubstateRequest, + GetSubstateResponse, + GetSubstatesRequest, + GetSubstatesResponse, + ListSubstatesRequest, + ListSubstatesResponse, +}; +use tari_ootle_common_types::SubstateRequirementRef; +use tari_validator_node_rpc::client::SubstateResult; + +use crate::rest_api::{context::HandlerContext, error::ErrorResponse, handlers::HandlerResult}; + +#[utoipa::path( + get, + path = "/substates", + description = "List substates, optionally filtered by type or template", + params( + ("filter_by_type" = Option, Query, description = "Filter substates by type"), + ("filter_by_template" = Option, Query, description = "Filter substates by template address"), + ("limit" = Option, Query, description = "Limit the number of results returned"), + ("offset" = Option, Query, description = "Offset the results by this amount"), + ), +)] +pub async fn list_substates( + Extension(context): Extension, + Query(req): Query, +) -> HandlerResult { + let ListSubstatesRequest { + filter_by_template, + filter_by_type, + limit, + offset, + } = req; + + let substates = context + .substate_manager() + .get_stored_substates_by_filters(filter_by_type, filter_by_template, limit, offset) + .map_err(|e| ErrorResponse::anyhow(anyhow!("Error getting substate: {}", e)))?; + + Ok(context.apply_cache_control(Json(ListSubstatesResponse { substates }), 100)) +} + +#[utoipa::path( + get, + path = "/substates/{substate_id}", + description = "Fetches a substate by ID", + params( + ("substate_id" = String, Path, description = "The substate ID to fetch"), + ("local_search_only" = bool, Query, description = "If true, only search local storage for the substate"), + ("version" = Option, Query, description = "Minimum version of the substate to fetch"), + ), +)] +pub async fn get_substate( + Extension(context): Extension, + Path(substate_id): Path, + Query(req): Query, +) -> HandlerResult> { + let maybe_substate = context + .substate_manager() + .get_substate(&substate_id, req.version) + .await + .map_err(|e| ErrorResponse::internal_error(format!("Error getting substate: {}", e)))?; + + match maybe_substate { + Some(substate_resp) => Ok(Json(GetSubstateResponse { + version: substate_resp.version, + substate: substate_resp.substate, + })), + None => { + if req.local_search_only { + Err(ErrorResponse::not_found(format!("Substate {} not found", substate_id))) + } else { + // Ask network + let substate = context + .transaction_manager() + .get_substate_from_network(SubstateRequirementRef::new(&substate_id, req.version)) + .await + .map_err(ErrorResponse::anyhow)?; + match substate { + SubstateResult::DoesNotExist => Err(ErrorResponse::not_found(format!( + "Substate {} not found on network", + substate_id + ))), + SubstateResult::Up { substate, .. } => Ok(Json(GetSubstateResponse { + version: substate.version(), + substate: substate.into_substate_value(), + })), + SubstateResult::Down { version, .. } => Err(ErrorResponse::not_found(format!( + "Substate {}v{} is DOWN on network", + substate_id, version + ))), + } + } + }, + } +} + +#[utoipa::path( + post, + path = "/substates/fetch", + description = "Fetches several substates by their IDs" +)] +pub async fn fetch_substates( + Extension(context): Extension, + Json(req): Json, +) -> HandlerResult> { + const MAX_REQUESTS: usize = 20; + + let GetSubstatesRequest { requests } = req; + + if requests.len() > MAX_REQUESTS { + return Err(ErrorResponse::bad_request(format!( + "Cannot request more than {MAX_REQUESTS} substates at once" + ))); + } + + let substates = context + .substate_manager() + .get_substates(requests.as_slice()) + .map_err(|e| ErrorResponse::internal_error(format!("Error getting substate: {}", e)))?; + + Ok(Json(GetSubstatesResponse { substates })) +} diff --git a/applications/tari_indexer/src/rest_api/handlers/templates.rs b/applications/tari_indexer/src/rest_api/handlers/templates.rs new file mode 100644 index 0000000000..665bd95cde --- /dev/null +++ b/applications/tari_indexer/src/rest_api/handlers/templates.rs @@ -0,0 +1,104 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use axum::{ + extract::{Path, Query}, + response::Response, + Extension, + Json, +}; +use tari_engine::{template::TemplateModuleLoader, wasm::WasmModule}; +use tari_indexer_client::types::{ + GetTemplateDefinitionResponse, + ListTemplatesRequest, + ListTemplatesResponse, + TemplateMetadata, +}; +use tari_ootle_common_types::optional::Optional; +use tari_ootle_storage::global::TemplateStatus; +use tari_template_lib::types::TemplateAddress; +use tari_template_manager::interface::{TemplateExecutable, TemplateManagerError}; + +use crate::rest_api::{context::HandlerContext, error::ErrorResponse, handlers::HandlerResult}; + +#[utoipa::path( + get, + path = "/templates/{template_id}", + description = "Fetch a template definition by its address" +)] +pub async fn get_template_definition( + Extension(context): Extension, + Path(template_address): Path, +) -> HandlerResult { + let template = context + .template_manager() + .fetch_template(&template_address) + .optional() + .map_err(|err| { + // If it's pending, we return a 404 to the client - this allows them to retry later + if matches!(err, TemplateManagerError::TemplateUnavailable { + status: Some(TemplateStatus::Pending) + }) { + ErrorResponse::not_found(format!( + "Template with address {} is still being downloaded. Try again later.", + template_address + )) + } else { + ErrorResponse::internal_error(format!("Error fetching template: {}", err)) + } + })? + .ok_or_else(|| ErrorResponse::not_found(format!("Template with address {} not found", template_address)))?; + + let template = match template.executable { + TemplateExecutable::CompiledWasm(code) => WasmModule::from_code(code) + .load_template() + .map_err(|e| ErrorResponse::internal_error(format!("Error loading template: {}", e)))?, + // TemplateExecutable::DownloadableWasm is never returned ad there is no DB type for that + TemplateExecutable::DownloadableWasm(_, _) | TemplateExecutable::Manifest(_) => { + return Err(ErrorResponse::bad_request("Template is not a wasm module".to_string())); + }, + }; + let resp = Json(GetTemplateDefinitionResponse { + definition: template.template_def().clone(), + name: template.template_name().to_string(), + }); + + Ok(context.apply_cache_control(resp, 1000)) +} + +#[utoipa::path( + get, + path = "/templates", + description = "List all templates", + params( + ("limit" = Option, Query, description = "Limit the number of results returned"), + ), +)] +pub async fn list_templates( + Extension(context): Extension, + Query(req): Query, +) -> HandlerResult { + let limit = req.limit.unwrap_or(10); + if limit == 0 || limit > 100 { + return Err(ErrorResponse::bad_request( + "Limit must be between 1 and 100".to_string(), + )); + } + + let templates = context + .template_manager() + .fetch_template_metadata(limit as usize) + .map_err(ErrorResponse::anyhow)?; + let resp = Json(ListTemplatesResponse { + templates: templates + .into_iter() + .map(|t| TemplateMetadata { + name: t.name, + address: t.address, + binary_sha: t.binary_sha, + }) + .collect(), + }); + + Ok(context.apply_cache_control(resp, 1000)) +} diff --git a/applications/tari_indexer/src/rest_api/handlers/transactions.rs b/applications/tari_indexer/src/rest_api/handlers/transactions.rs new file mode 100644 index 0000000000..54104c90a7 --- /dev/null +++ b/applications/tari_indexer/src/rest_api/handlers/transactions.rs @@ -0,0 +1,147 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use axum::{ + extract::{Path, Query}, + response::Response, + Extension, + Json, +}; +use log::*; +use tari_consensus_types::Decision; +use tari_indexer_client::types::{ + GetTransactionResultResponse, + IndexerTransactionFinalizedResult, + ListRecentTransactionsRequest, + ListRecentTransactionsResponse, + SubmitTransactionRequest, + SubmitTransactionResponse, +}; +use tari_ootle_common_types::optional::Optional; +use tari_transaction::TransactionId; +use tari_validator_node_rpc::client::TransactionResultStatus; +use time::{OffsetDateTime, PrimitiveDateTime}; + +use crate::{ + network_client::NetworkClientError, + rest_api::{context::HandlerContext, error::ErrorResponse, handlers::HandlerResult}, + transaction_manager::error::TransactionManagerError, +}; + +const LOG_TARGET: &str = "tari::indexer::rest_api::handlers::transactions"; + +#[utoipa::path( + post, + path = "/transactions", + description = "Submit a transaction to validators responsible for the involved shards, or run the transaction as \ + a dry-run" +)] +pub async fn submit_transaction( + Extension(context): Extension, + Json(req): Json, +) -> HandlerResult> { + let request: SubmitTransactionRequest = req; + + if request.is_dry_run { + let transaction_id = request.transaction.calculate_id(); + let exec_result = context + .dry_run_transaction_processor() + .process_transaction(request.transaction) + .await + .map_err(ErrorResponse::anyhow)?; + + return Ok(Json(SubmitTransactionResponse { + result: IndexerTransactionFinalizedResult::Finalized { + execution_result: Some(Box::new(exec_result)), + final_decision: Decision::Commit, + abort_details: None, + finalized_time: now(), + execution_time: Default::default(), + }, + transaction_id, + })); + } + + let transaction = request.transaction; + let transaction_id = context + .transaction_manager() + .submit_transaction(transaction) + .await + .map_err(|e| match e { + TransactionManagerError::NetworkClientError(NetworkClientError::AllValidatorsFailed { .. }) => { + ErrorResponse::service_unavailable(format!("All validators failed: {}", e)) + }, + TransactionManagerError::InvalidTransaction { + transaction_id, + details, + } => ErrorResponse::bad_request(format!("Transaction {} is invalid: {}", transaction_id, details)), + e => ErrorResponse::anyhow(e), + })?; + + info!(target: LOG_TARGET, "✅ Transaction submitted: {}", transaction_id); + + Ok(Json(SubmitTransactionResponse { + result: IndexerTransactionFinalizedResult::Pending, + transaction_id, + })) +} + +fn now() -> PrimitiveDateTime { + let now = OffsetDateTime::now_utc(); + PrimitiveDateTime::new(now.date(), now.time()) +} + +#[utoipa::path(get, path = "/transactions/recent", description = "List recent transactions")] +pub async fn list_recent_transactions( + Extension(context): Extension, + Query(req): Query, +) -> HandlerResult { + let limit = req.limit.unwrap_or(100); + if limit > 100 { + return Err(ErrorResponse::bad_request( + "Limit cannot be greater than 100".to_string(), + )); + } + + let transactions = context + .transaction_manager() + .list_recent_transactions(req.last_id, limit as usize) + .map_err(ErrorResponse::anyhow)?; + + Ok(context.apply_cache_control(Json(ListRecentTransactionsResponse { transactions }), 30)) +} + +#[utoipa::path( + post, + path = "/transactions/{transaction_id}/result", + description = "Get the result of a submitted transaction (by transaction ID)" +)] +pub async fn get_transaction_result( + Extension(context): Extension, + Path(transaction_id): Path, +) -> HandlerResult> { + let result = context + .transaction_manager() + .get_transaction_result(transaction_id) + .await + .optional() + .map_err(ErrorResponse::anyhow)? + .ok_or_else(|| ErrorResponse::not_found(format!("Transaction {transaction_id} not found")))?; + + let resp = match result { + TransactionResultStatus::Pending => GetTransactionResultResponse { + result: IndexerTransactionFinalizedResult::Pending, + }, + TransactionResultStatus::Finalized(finalized) => GetTransactionResultResponse { + result: IndexerTransactionFinalizedResult::Finalized { + final_decision: finalized.final_decision, + execution_result: finalized.execute_result.map(Box::new), + execution_time: finalized.execution_time, + finalized_time: finalized.finalized_time, + abort_details: finalized.abort_details, + }, + }, + }; + + Ok(Json(GetTransactionResultResponse { result: resp.result })) +} diff --git a/applications/tari_indexer/src/rest_api/handlers/utxos.rs b/applications/tari_indexer/src/rest_api/handlers/utxos.rs new file mode 100644 index 0000000000..845b8e9e4c --- /dev/null +++ b/applications/tari_indexer/src/rest_api/handlers/utxos.rs @@ -0,0 +1,106 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::collections::HashSet; + +use axum::{ + http::header::HeaderMap, + response::{IntoResponse, Response}, + Extension, + Json, +}; +use log::*; +use tari_indexer_client::types::{GetUnspentUtxosRequest, GetUnspentUtxosResponse, GetUtxoUpdatesRequest}; +use tari_ootle_common_types::NumPreshards; + +use crate::rest_api::{ + context::HandlerContext, + error::ErrorResponse, + handlers::HandlerResult, + streaming::{encoding, UtxoUpdateStream}, +}; + +const LOG_TARGET: &str = "tari::ootle::indexer::rest_api::handlers::utxos"; + +#[utoipa::path(post, path = "/utxos/updates", description = "Streams UTXO updates")] +pub async fn stream_utxo_updates( + headers: HeaderMap, + Extension(context): Extension, + Json(req): Json, +) -> HandlerResult { + if req.per_shard_limit > 1000 { + return Err(ErrorResponse::bad_request( + "per_shard_limit cannot be greater than 1000", + )); + } + + if req.shard_state_versions.len() > NumPreshards::MAX_SHARD.as_u32() as usize { + return Err(ErrorResponse::bad_request(format!( + "shard_state_versions cannot contain more than {} entries", + NumPreshards::MAX_SHARD.as_u32() + ))); + } + + if req + .shard_state_versions + .iter() + .any(|(shard, _)| shard.is_global() || *shard > NumPreshards::MAX_SHARD) + { + return Err(ErrorResponse::bad_request(format!( + "shard_state_versions contains invalid shard. Cannot query UTXOs in global shard or greater than max \ + shard {} exceeded", + NumPreshards::MAX_SHARD.as_u32() + ))); + } + + info!( + target: LOG_TARGET, + "Starting UTXO update stream for {} shard(s)", + req.shard_state_versions.len() + ); + + let accept_header = headers.get("accept").and_then(|v| v.to_str().ok()); + let encoder = match accept_header { + Some(mime) => encoding::from_media_type(mime).ok_or_else(|| { + ErrorResponse::bad_request(format!( + "Unsupported Accept header '{}', supported types are 'application/x-protobuf' and 'application/json'", + mime + )) + })?, + None => encoding::MimeTypeEncoder::protobuf(), + }; + + // Check for duplicate shards + let mut seen = HashSet::new(); + for (shard, _) in &req.shard_state_versions { + if !seen.insert(shard) { + return Err(ErrorResponse::bad_request(format!( + "Duplicate shard {} in shard_state_versions", + shard.as_u32() + ))); + } + } + + let stream = UtxoUpdateStream::new(context.substate_manager().clone(), req, encoder); + Ok(stream.into_response()) +} + +#[utoipa::path( + post, + path = "/utxos/fetch", + description = "Gets full UTXO data for a list of UTXO IDs" +)] +pub async fn fetch_utxos( + Extension(context): Extension, + Json(req): Json, +) -> HandlerResult> { + if req.tag_and_nonce_pairs.len() > 1000 { + return Err(ErrorResponse::bad_request("cannot query more than 1000 UTXOs")); + } + let utxos = context + .substate_manager() + .get_unspent_utxos(&req.resource_address, &req.tag_and_nonce_pairs) + .map_err(ErrorResponse::anyhow)?; + + Ok(Json(GetUnspentUtxosResponse { utxos })) +} diff --git a/applications/tari_indexer/src/rest_api/mod.rs b/applications/tari_indexer/src/rest_api/mod.rs new file mode 100644 index 0000000000..7015e9e7a0 --- /dev/null +++ b/applications/tari_indexer/src/rest_api/mod.rs @@ -0,0 +1,12 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +mod cache; +mod context; +mod encoder; +mod error; +mod handlers; +mod server; +mod streaming; + +pub use server::*; diff --git a/applications/tari_indexer/src/rest_api/server.rs b/applications/tari_indexer/src/rest_api/server.rs new file mode 100644 index 0000000000..f15988eda5 --- /dev/null +++ b/applications/tari_indexer/src/rest_api/server.rs @@ -0,0 +1,108 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::net::SocketAddr; + +use axum::{ + routing::{get, post}, + Extension, + Router, +}; +use log::*; +use tari_ootle_app_utilities::tcp::try_bind_with_fallback; +use tari_shutdown::ShutdownSignal; +use tower_http::{cors::CorsLayer, limit::RequestBodyLimitLayer}; +use utoipa::OpenApi; +use utoipa_swagger_ui::SwaggerUi; + +use crate::{ + bootstrap::Services, + rest_api::{context::HandlerContext, handlers}, +}; + +const LOG_TARGET: &str = "tari::ootle::indexer::rest_api::server"; + +// Limit the body size to 4MB to allow for large transactions (wasm uploads) +const REQUEST_BODY_LIMIT: usize = 4 * 1024 * 1024; // 4 MB + +#[derive(OpenApi)] +#[openapi(paths( + handlers::misc::get_identity, + handlers::misc::wait_until_ready, + handlers::misc::get_epoch_manager_stats, + handlers::network::get_network_sync_stats, + handlers::network::get_connections, + handlers::network::add_connection, + handlers::substates::get_substate, + handlers::substates::list_substates, + handlers::substates::fetch_substates, + handlers::transactions::submit_transaction, + handlers::transactions::list_recent_transactions, + handlers::transactions::get_transaction_result, + handlers::templates::get_template_definition, + handlers::templates::list_templates, + handlers::utxos::fetch_utxos, + handlers::utxos::stream_utxo_updates, +))] +pub struct ApiDoc; + +pub struct Server; + +impl Server { + pub async fn spawn( + preferred_addr: SocketAddr, + services: &Services, + shutdown: ShutdownSignal, + ) -> anyhow::Result { + let context = HandlerContext::from_services(services); + + let router = Router::new() + .route("/identity", get(handlers::misc::get_identity)) + .route("/wait-until-ready", get(handlers::misc::wait_until_ready)) + .route("/epoch-manager/stats", get(handlers::misc::get_epoch_manager_stats)) + .route("/network/stats", get(handlers::network::get_network_sync_stats)) + .route( + "/network/connections", + get(handlers::network::get_connections).post(handlers::network::add_connection), + ) + .route("/substates/{substate_id}", get(handlers::substates::get_substate)) + .route( + "/substates", + get(handlers::substates::list_substates).post(handlers::substates::fetch_substates), + ) + .route("/transactions", post(handlers::transactions::submit_transaction)) + .route( + "/transactions/recent", + get(handlers::transactions::list_recent_transactions), + ) + .route( + "/transactions/{transaction_id}/result", + get(handlers::transactions::get_transaction_result), + ) + .route( + "/templates/{template_address}", + get(handlers::templates::get_template_definition), + ) + .route("/templates", get(handlers::templates::list_templates)) + .route("/non-fungibles", get(handlers::nfts::get_non_fungibles)) // Placeholder for future non-fungible endpoints + .route("/utxos/fetch", post(handlers::utxos::fetch_utxos)) + .route("/utxos/stream", post(handlers::utxos::stream_utxo_updates)) + .layer(CorsLayer::permissive()) + .layer(RequestBodyLimitLayer::new(REQUEST_BODY_LIMIT)) + .merge(SwaggerUi::new("/swagger-ui").url("/openapi.json", ApiDoc::openapi())) + .layer(Extension(context)); + + let listener = try_bind_with_fallback(preferred_addr).await?; + + // spawn server + let listen_addr = listener.local_addr()?; + info!(target: LOG_TARGET, "🌐 Indexer REST API server listening on {listen_addr}"); + tokio::spawn(async move { + if let Err(error) = axum::serve(listener, router).with_graceful_shutdown(shutdown).await { + error!(target: LOG_TARGET, "Wallet query HTTP server error: {error}"); + } + }); + + Ok(listen_addr) + } +} diff --git a/applications/tari_indexer/src/rest_api/streaming/encoding.rs b/applications/tari_indexer/src/rest_api/streaming/encoding.rs new file mode 100644 index 0000000000..8586cb94df --- /dev/null +++ b/applications/tari_indexer/src/rest_api/streaming/encoding.rs @@ -0,0 +1,88 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::{io::Write, str::FromStr}; + +use bytes::{BufMut, Bytes}; +use log::*; +use mediatype::MediaType; +use prost::Message; +use tari_indexer_client::protobuf::UtxoUpdatePayload; + +use crate::rest_api::encoder::Encoder; + +const LOG_TARGET: &str = "tari::indexer::rest_api::streaming::encoding"; + +pub struct ProtobufEncoder; + +impl Encoder for ProtobufEncoder { + type Item = UtxoUpdatePayload; + + fn encode_into(&self, msg: &Self::Item, buf: &mut impl BufMut) -> anyhow::Result<()> { + // Length-delimited protobuf + if log_enabled!(Level::Trace) { + let len = msg.encoded_len(); + trace!(target: LOG_TARGET, "🚧 Encoding protobuf message of length: {}", len); + } + let len = msg.encoded_len(); + debug!(target: LOG_TARGET, "🚧 Encoding protobuf message of length: {}", len); + msg.encode_length_delimited(buf)?; + Ok(()) + } +} + +pub struct JsonEncoder; + +impl Encoder for JsonEncoder { + type Item = UtxoUpdatePayload; + + fn encode_into(&self, msg: &Self::Item, buf: &mut impl BufMut) -> anyhow::Result<()> { + let json = serde_json::to_vec(msg)?; + buf.put(Bytes::from(json)); + // Line delimited + buf.writer().write_all(b"\n")?; + Ok(()) + } +} + +pub enum MimeTypeEncoder { + Protobuf(ProtobufEncoder), + Json(JsonEncoder), +} + +impl MimeTypeEncoder { + pub fn protobuf() -> Self { + MimeTypeEncoder::Protobuf(ProtobufEncoder) + } +} + +impl Encoder for MimeTypeEncoder { + type Item = UtxoUpdatePayload; + + fn encode_into(&self, msg: &Self::Item, buf: &mut impl BufMut) -> anyhow::Result<()> { + match self { + MimeTypeEncoder::Protobuf(encoder) => encoder.encode_into(msg, buf), + MimeTypeEncoder::Json(encoder) => encoder.encode_into(msg, buf), + } + } +} + +pub fn from_media_type(mime_type: &str) -> Option { + const ACCEPTED_TYPES: &[MediaType] = &[ + MediaType::new( + mediatype::Name::new_unchecked("application"), + mediatype::Name::new_unchecked("x-protobuf"), + ), + MediaType::new( + mediatype::Name::new_unchecked("application"), + mediatype::Name::new_unchecked("json"), + ), + ]; + let media_type = headers_accept::Accept::from_str(mime_type).ok()?; + let accepted = media_type.negotiate(ACCEPTED_TYPES)?; + match accepted.subty.as_str() { + "x-protobuf" => Some(MimeTypeEncoder::Protobuf(ProtobufEncoder)), + "json" => Some(MimeTypeEncoder::Json(JsonEncoder)), + _ => None, + } +} diff --git a/applications/tari_indexer/src/rest_api/streaming/mod.rs b/applications/tari_indexer/src/rest_api/streaming/mod.rs new file mode 100644 index 0000000000..1c25ce8adb --- /dev/null +++ b/applications/tari_indexer/src/rest_api/streaming/mod.rs @@ -0,0 +1,7 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +pub mod encoding; +mod utxo_stream; + +pub use utxo_stream::*; diff --git a/applications/tari_indexer/src/rest_api/streaming/utxo_stream.rs b/applications/tari_indexer/src/rest_api/streaming/utxo_stream.rs new file mode 100644 index 0000000000..c76c0668af --- /dev/null +++ b/applications/tari_indexer/src/rest_api/streaming/utxo_stream.rs @@ -0,0 +1,253 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use axum::response::{IntoResponse, Response}; +use bytes::{Bytes, BytesMut}; +use futures::Stream; +use log::*; +use tari_indexer_client::{protobuf, types::GetUtxoUpdatesRequest}; +use tari_ootle_common_types::{shard::Shard, StateVersion}; +use tari_ootle_wallet_sdk::models::WalletUtxoUpdate; + +use crate::{ + rest_api::{encoder::Encoder, error::ErrorResponse, streaming::encoding::MimeTypeEncoder}, + substate_manager::SubstateManager, +}; + +const LOG_TARGET: &str = "tari::indexer::rest_api::streaming::utxo_stream"; + +pub struct UtxoUpdateStream { + substate_manager: SubstateManager, + request: GetUtxoUpdatesRequest, + requested_shard_index: usize, + buffer: BytesMut, + pending_updates: Option, + is_done: bool, + encoder: MimeTypeEncoder, +} + +impl UtxoUpdateStream { + pub fn new(substate_manager: SubstateManager, request: GetUtxoUpdatesRequest, encoder: MimeTypeEncoder) -> Self { + Self { + substate_manager, + request, + requested_shard_index: 0, + pending_updates: None, + buffer: BytesMut::with_capacity(1024), + is_done: false, + encoder, + } + } + + fn current_requested_state_shard(&self) -> Option<(Shard, StateVersion)> { + self.request + .shard_state_versions + .get(self.requested_shard_index) + .copied() + } + + fn encode_to_buffer(&mut self) -> anyhow::Result<()> { + let mut pending_updates = self + .pending_updates + .take() + .expect("BUG: no pending updates in encode_to_buffer"); + + let mut payload = protobuf::UtxoUpdatePayload::default(); + + if !pending_updates.sos_emitted { + payload.sos = Some(protobuf::StartOfShard { + shard: pending_updates.shard.as_u32(), + max_state_version: pending_updates.updates_state_version.as_u64(), + num_updates: u32::try_from(pending_updates.updates_len()).expect("loaded more than u32::MAX updates"), + }); + pending_updates.sos_emitted = true; + } + + if let Some(update) = pending_updates.next() { + payload.update = Some(protobuf::WalletUtxoUpdate::from(update)); + } + + if pending_updates.is_empty() { + // No more updates in this batch + payload.eos = Some(protobuf::EndOfShard { + max_state_version: pending_updates.high_watermark_state_version.as_u64(), + }); + } + + // This should be a max of about 60 bytes for protobuf (observed max 54 bytes) + self.encoder.encode_into(&payload, &mut self.buffer)?; + + debug!( + target: LOG_TARGET, + "Encoded payload size: {}, pending updates remaining in batch: {}", + self.buffer.len(), + pending_updates.updates_len() + ); + + if payload.eos.is_none() { + // Put pending updates back for next call + self.pending_updates = Some(pending_updates); + } + + Ok(()) + } + + pub fn next_batch(&mut self, shard: Shard, state_version: StateVersion) -> anyhow::Result { + let (updates_state_version, updates) = self.substate_manager.get_utxo_updates( + self.request.resource_address, + shard, + state_version, + self.request.unspent_only, + self.request.per_shard_limit, + )?; + let high_watermark_state_version = self + .substate_manager + .get_max_state_version(&self.request.resource_address, shard)?; + if updates.is_empty() { + return Ok(false); + } + debug!( + target: LOG_TARGET, + "Received {} updates for shard {}, max_state_version {} -> {}", + updates.len(), + shard, + updates_state_version, + high_watermark_state_version + ); + self.pending_updates = Some(PendingUpdates { + sos_emitted: false, + shard, + updates_state_version, + high_watermark_state_version, + updates, + index: 0, + }); + + Ok(true) + } +} + +impl Stream for UtxoUpdateStream { + type Item = anyhow::Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + if self.is_done { + return Poll::Ready(None); + } + + let this = self.get_mut(); + loop { + if this.pending_updates.is_none() { + let (shard, state_version) = match this.current_requested_state_shard() { + Some(v) => v, + None => { + debug!(target: LOG_TARGET, "No more shards to request"); + this.is_done = true; + return Poll::Ready(None); + }, + }; + + debug!( + target: LOG_TARGET, + "Requesting UTXO updates for shard {} from state version {}", + shard, + state_version + ); + + // Get more updates + match this.next_batch(shard, state_version) { + Ok(true) => { + // start sending updates + }, + Ok(false) => { + // No updates in this shard, move to next + this.requested_shard_index += 1; + debug!( + target: LOG_TARGET, + "No updates found for shard {}, moving to next shard", + shard + ); + continue; + }, + Err(e) => { + error!(target: LOG_TARGET, "Error fetching UTXO updates: {}", e); + this.is_done = true; + break Poll::Ready(Some(Err(anyhow::anyhow!(e)))); + }, + } + } + + if let Err(e) = this.encode_to_buffer() { + error!(target: LOG_TARGET, "Error encoding UTXO update: {}", e); + this.is_done = true; + return Poll::Ready(Some(Err(anyhow::anyhow!(e)))); + } + + debug!( + target: LOG_TARGET, + "Buffer size after encoding: {} bytes", + this.buffer.len() + ); + + if this.pending_updates.is_none() { + // Finished this shard, move to next + // Note that the client has requested a maximum number of updates per shard, so we may not have sent all + // updates, but it is up to the client to request again with updated state versions + this.requested_shard_index += 1; + } + + // Drain and send the buffer + break Poll::Ready(Some(Ok(this.buffer.split().freeze()))); + } + } +} + +impl IntoResponse for UtxoUpdateStream { + fn into_response(self) -> Response { + let stream_body = axum::body::Body::from_stream(self); + + axum::response::Response::builder() + .header("Content-Type", "application/octet-stream") + .body(stream_body) + .map_err(ErrorResponse::anyhow) + .into_response() + } +} + +struct PendingUpdates { + pub sos_emitted: bool, + pub shard: Shard, + pub updates_state_version: StateVersion, + pub high_watermark_state_version: StateVersion, + pub updates: Vec, + pub index: usize, +} + +impl PendingUpdates { + #[allow(dead_code)] + pub fn is_done(&self) -> bool { + self.index >= self.updates.len() + } + + pub fn next(&mut self) -> Option<&WalletUtxoUpdate> { + if self.is_done() { + return None; + } + let update = &self.updates[self.index]; + self.index += 1; + Some(update) + } + + pub const fn updates_len(&self) -> usize { + self.updates.len() - self.index + } + + pub const fn is_empty(&self) -> bool { + self.updates_len() == 0 + } +} diff --git a/applications/tari_indexer/src/storage_sqlite/reader.rs b/applications/tari_indexer/src/storage_sqlite/reader.rs index 62049bc190..7c6ac1c15b 100644 --- a/applications/tari_indexer/src/storage_sqlite/reader.rs +++ b/applications/tari_indexer/src/storage_sqlite/reader.rs @@ -389,17 +389,26 @@ impl IndexerStoreReadTransaction for SqliteStoreReadTransaction<'_> { resource_address: ResourceAddress, shard: Shard, from_state_version: StateVersion, + unspent_only: bool, limit: u32, ) -> Result<(StateVersion, Vec), StorageError> { const OPERATION: &str = "get_utxo_updates"; use crate::storage_sqlite::schema::utxos; - let rows = utxos::table + let mut query = utxos::table .filter(utxos::resource_address.eq(resource_address.to_string())) .filter(utxos::state_version.gt(from_state_version.as_u64() as i64)) .filter(utxos::shard.eq(shard.as_u32() as i32)) .limit(i64::from(limit)) .order_by(utxos::state_version.asc()) + .into_boxed(); + if unspent_only { + // Only return unspent UTXOs + query = query + .filter(utxos::is_spent.eq(false)) + .filter(utxos::is_burnt.eq(false)); + } + let rows = query .load_iter::(self.connection()) .map_err(|e| StorageError::QueryError { reason: format!("{OPERATION}: {}", e), diff --git a/applications/tari_indexer/src/storage_sqlite/store_factory.rs b/applications/tari_indexer/src/storage_sqlite/store_factory.rs index a43ddc4ea2..d4d32ae599 100644 --- a/applications/tari_indexer/src/storage_sqlite/store_factory.rs +++ b/applications/tari_indexer/src/storage_sqlite/store_factory.rs @@ -185,6 +185,7 @@ pub trait IndexerStoreReadTransaction { resource_address: ResourceAddress, shard: Shard, from_state_version: StateVersion, + unspents_only: bool, limit: u32, ) -> Result<(StateVersion, Vec), StorageError>; diff --git a/applications/tari_indexer/src/substate_manager.rs b/applications/tari_indexer/src/substate_manager.rs index 838d03500b..9c8cd2cee2 100644 --- a/applications/tari_indexer/src/substate_manager.rs +++ b/applications/tari_indexer/src/substate_manager.rs @@ -115,11 +115,12 @@ impl SubstateManager { resource_address: ResourceAddress, shard: Shard, from_state_version: StateVersion, + unspent_only: bool, limit: u32, ) -> Result<(StateVersion, Vec), anyhow::Error> { - let updates = self - .substate_store - .with_read_tx(|tx| tx.utxos_get_updates(resource_address, shard, from_state_version, limit))?; + let updates = self.substate_store.with_read_tx(|tx| { + tx.utxos_get_updates(resource_address, shard, from_state_version, unspent_only, limit) + })?; Ok(updates) } @@ -147,17 +148,17 @@ impl SubstateManager { pub async fn get_substate( &self, - substate_address: &SubstateId, + substate_id: &SubstateId, version: Option, ) -> Result, anyhow::Error> { // we store the latest version of the substates related to the events // so we will return the substate directly from database if it's there - if let Some(substate) = self.get_substate_from_db(substate_address, version).await? { + if let Some(substate) = self.get_substate_from_db(substate_id, version).await? { return Ok(Some(substate)); } // the substate is not in db (or is not the requested version) so we fetch it from the dan layer committee - let substate_result = self.substate_scanner.get_substate(substate_address, version).await?; + let substate_result = self.substate_scanner.get_substate(substate_id, version).await?; match substate_result { SubstateResult::Up { id, substate } => Ok(Some(SubstateResponse { address: id, diff --git a/applications/tari_indexer/src/transaction_manager/mod.rs b/applications/tari_indexer/src/transaction_manager/mod.rs index dfce09e845..a50861c1c7 100644 --- a/applications/tari_indexer/src/transaction_manager/mod.rs +++ b/applications/tari_indexer/src/transaction_manager/mod.rs @@ -27,7 +27,7 @@ use tari_indexer_client::types::TransactionEntry; use tari_ootle_common_types::{ optional::{IsNotFoundError, Optional}, NodeAddressable, - SubstateRequirement, + SubstateRequirementRef, ToSubstateAddress, }; use tari_transaction::{Transaction, TransactionId}; @@ -44,6 +44,7 @@ use crate::{ transaction_manager::error::TransactionManagerError, }; +#[derive(Debug, Clone)] pub struct TransactionManager { network_client: TariNetworkClient, store: TStore, @@ -92,15 +93,15 @@ where }) } - pub async fn get_substate( + pub async fn get_substate_from_network( &self, - substate_requirement: &SubstateRequirement, + substate_requirement: SubstateRequirementRef<'_>, ) -> Result { - let address = substate_requirement.to_substate_address_zero_version(); + let address = substate_requirement.or_zero_version().to_substate_address(); let result = self .network_client .try_single_with_committee(address, |mut client| async move { - client.get_substate(substate_requirement.as_ref()).await + client.get_substate(substate_requirement).await }) .await?; Ok(result) diff --git a/applications/tari_indexer/web_ui/package.json b/applications/tari_indexer/web_ui/package.json index 11b4e31516..41b8b4318b 100644 --- a/applications/tari_indexer/web_ui/package.json +++ b/applications/tari_indexer/web_ui/package.json @@ -18,6 +18,7 @@ "@tanstack/react-query": "^5.81.2", "@tanstack/react-query-devtools": "^5.84.2", "@tari-project/typescript-bindings": "link:../../../bindings", + "@tari-project/indexer-client": "link:../../../clients/javascript/indexer_client", "file-saver": "^2.0.5", "react": "^19.1.0", "react-dom": "^19.1.0", diff --git a/applications/tari_indexer/web_ui/src/api/hooks/useSubstates.tsx b/applications/tari_indexer/web_ui/src/api/hooks/useSubstates.tsx index d6e00cae91..e51d89e1dc 100644 --- a/applications/tari_indexer/web_ui/src/api/hooks/useSubstates.tsx +++ b/applications/tari_indexer/web_ui/src/api/hooks/useSubstates.tsx @@ -21,7 +21,7 @@ // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import { useQuery } from "@tanstack/react-query"; -import { listSubstates, getSubstate, getNonFungibles } from "../../utils/json_rpc"; +import { listSubstates, getSubstate, getNonFungibles } from "../../utils/api"; interface UseListSubstatesProps { offset: number; @@ -54,21 +54,22 @@ interface UseGetSubstateProps { enabled?: boolean; } -export const useGetSubstate = ({ - address, - version = null, - local_search_only = false, - enabled = true, -}: UseGetSubstateProps) => { +export const useGetSubstate = (props: UseGetSubstateProps) => { + const { + address, + version = null, + local_search_only = false, + enabled = true, + } = props; return useQuery({ queryKey: ["substate", address, version, local_search_only], queryFn: async () => { // @ts-ignore - return await getSubstate({ + return await getSubstate( address, version, local_search_only, - }); + ); }, enabled: enabled && !!address, staleTime: 5 * 60 * 1000, @@ -83,11 +84,11 @@ interface UseGetNonFungiblesProps { } export const useGetNonFungibles = ({ - address, - start_index = 0, - end_index = 10, - enabled = true, -}: UseGetNonFungiblesProps) => { + address, + start_index = 0, + end_index = 10, + enabled = true, + }: UseGetNonFungiblesProps) => { return useQuery({ queryKey: ["nonFungibles", address, start_index, end_index], queryFn: async () => { diff --git a/applications/tari_indexer/web_ui/src/api/hooks/useTransactions.tsx b/applications/tari_indexer/web_ui/src/api/hooks/useTransactions.tsx index d06ac702f7..261688f62b 100644 --- a/applications/tari_indexer/web_ui/src/api/hooks/useTransactions.tsx +++ b/applications/tari_indexer/web_ui/src/api/hooks/useTransactions.tsx @@ -24,7 +24,7 @@ import { useQuery } from "@tanstack/react-query"; import { listRecentTransactions, getTransactionResult, -} from "../../utils/json_rpc"; +} from "../../utils/api"; interface UseListRecentTransactionsProps { last_id: string | null; @@ -32,9 +32,9 @@ interface UseListRecentTransactionsProps { } export const useListRecentTransactions = ({ - last_id, - limit, -}: UseListRecentTransactionsProps) => { + last_id, + limit, + }: UseListRecentTransactionsProps) => { return useQuery({ queryKey: ["recent_transactions"], queryFn: () => { diff --git a/applications/tari_indexer/web_ui/src/routes/VN/Components/Connections.tsx b/applications/tari_indexer/web_ui/src/routes/VN/Components/Connections.tsx index 9632e2847c..617fb3ecf4 100644 --- a/applications/tari_indexer/web_ui/src/routes/VN/Components/Connections.tsx +++ b/applications/tari_indexer/web_ui/src/routes/VN/Components/Connections.tsx @@ -21,7 +21,7 @@ // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import React, { useCallback, useEffect, useRef, useState } from "react"; -import { addPeer, getConnections } from "../../../utils/json_rpc"; +import { addPeer, getConnections } from "../../../utils/api"; import { shortenString } from "./helpers"; import Table from "@mui/material/Table"; import TableBody from "@mui/material/TableBody"; @@ -150,14 +150,14 @@ function Connections() { {connections && connections.map( ({ - connection_id, - address, - age, - direction, - peer_id, - ping_latency, - user_agent, - }) => ( + connection_id, + address, + age, + direction, + peer_id, + ping_latency, + user_agent, + }) => ( {peer_id ? shortenString(peer_id) : "--"} @@ -173,7 +173,7 @@ function Connections() { {user_agent ? user_agent.replace(/^\/tari\//, "") : "--"} - ) + ), )} diff --git a/applications/tari_indexer/web_ui/src/routes/VN/ValidatorNode.tsx b/applications/tari_indexer/web_ui/src/routes/VN/ValidatorNode.tsx index 3de297b0cb..eb25ab8481 100644 --- a/applications/tari_indexer/web_ui/src/routes/VN/ValidatorNode.tsx +++ b/applications/tari_indexer/web_ui/src/routes/VN/ValidatorNode.tsx @@ -23,7 +23,7 @@ import { useEffect, useState } from "react"; import Connections from "./Components/Connections"; import Info from "./Components/Info"; -import { getIdentity } from "../../utils/json_rpc"; +import { getIdentity } from "../../utils/api"; import RecentTransactions from "../RecentTransactions/components/RecentTransactions"; import "./ValidatorNode.css"; import { StyledPaper } from "../../Components/StyledComponents"; @@ -42,7 +42,7 @@ function ValidatorNode() { }) .catch((reason) => { console.log(reason); - setError("Json RPC error, please check console"); + setError("API error, please check the developer console"); }); }, []); useEffect(() => { diff --git a/applications/tari_indexer/web_ui/src/utils/json_rpc.tsx b/applications/tari_indexer/web_ui/src/utils/api.tsx similarity index 50% rename from applications/tari_indexer/web_ui/src/utils/json_rpc.tsx rename to applications/tari_indexer/web_ui/src/utils/api.tsx index 6a28894351..2f78c60a18 100644 --- a/applications/tari_indexer/web_ui/src/utils/json_rpc.tsx +++ b/applications/tari_indexer/web_ui/src/utils/api.tsx @@ -23,42 +23,30 @@ import type { IndexerAddPeerRequest, IndexerAddPeerResponse, - // IndexerGetAllVnsResponse, - IndexerGetCommsStatsResponse, IndexerGetConnectionsResponse, - GetEpochManagerStatsResponse, IndexerGetIdentityResponse, - // GetNonFungibleCollectionsResponse, - // GetNonFungibleCountRequest, - // GetNonFungibleCountResponse, GetNonFungiblesRequest, GetNonFungiblesResponse, - // GetRelatedTransactionsRequest, - // GetRelatedTransactionsResponse, - IndexerGetSubstateRequest, IndexerGetSubstateResponse, IndexerGetTransactionResultRequest, IndexerGetTransactionResultResponse, ListRecentTransactionsRequest, ListRecentTransactionsResponse, - InspectSubstateRequest, - InspectSubstateResponse, - IndexerSubmitTransactionResponse, ListSubstatesRequest, - ListSubstatesResponse, - // SubstateType, + ListSubstatesResponse, SubstateId, } from "@tari-project/typescript-bindings"; +import { IndexerClient } from "@tari-project/indexer-client"; -const DEFAULT_WALLET_ADDRESS = new URL( - import.meta.env.VITE_INDEXER_JRPC_ADDRESS || - import.meta.env.VITE_JSON_RPC_ADDRESS || - import.meta.env.VITE_JRPC_ADDRESS || - "http://localhost:9000" +const DEFAULT_API_ADDRESS = new URL( + import.meta.env.VITE_INDEXER_API_ADDRESS || + import.meta.env.VITE_API_ADDRESS || + import.meta.env.VITE_API_ADDRESS || + "http://localhost:9000", ); export async function getClientAddress(): Promise { try { - const resp = await fetch("/json_rpc_address"); + const resp = await fetch("/rest_api_address"); if (resp.status === 200) { const url = await resp.text(); try { @@ -71,70 +59,62 @@ export async function getClientAddress(): Promise { console.warn(e); } - return DEFAULT_WALLET_ADDRESS; + return DEFAULT_API_ADDRESS; } -async function jsonRpc(method: string, params: any = null) { - let id = 0; - id += 1; - let address = await getClientAddress(); - let response = await fetch(address, { - method: "POST", - body: JSON.stringify({ - method: method, - jsonrpc: "2.0", - id: id, - params: params, - }), - headers: { - "Content-Type": "application/json", - }, - }); - let json = await response.json(); - if (json.error) { - throw json.error; +let clientInstance: IndexerClient | null = null; +let pendingClientInstance: Promise | null = null; +let outerAddress: URL | null = null; + +export async function client() { + if (clientInstance) { + return Promise.resolve(clientInstance); } - return json.result; + + const getAddress = outerAddress ? Promise.resolve(outerAddress) : getClientAddress(); + + pendingClientInstance = getAddress.then(async (addr) => { + const client = IndexerClient.usingFetchTransport(addr.toString()); + outerAddress = addr; + clientInstance = client; + pendingClientInstance = null; + return client; + }); + + return pendingClientInstance; } -export const getOpenRpcSchema = (): Promise => jsonRpc("rpc.discover"); + export const getIdentity = (): Promise => - jsonRpc("get_identity"); + client().then((c) => c.identityGet()); export const addPeer = ( - request: IndexerAddPeerRequest -): Promise => jsonRpc("add_peer", request); -export const getCommsStats = (): Promise => - jsonRpc("get_comms_stats"); + request: IndexerAddPeerRequest, +): Promise => client().then((c) => c.addConnection(request)); +export const getConnections = (): Promise => + client().then((c) => c.getConnections()); + export const getSubstate = ( - request: IndexerGetSubstateRequest -): Promise => jsonRpc("get_substate", request); -export const inspectSubstate = ( - request: InspectSubstateRequest -): Promise => jsonRpc("inspect_substate", request); + id: SubstateId, + version?: number | null, + local_search_only?: boolean, +): Promise => client().then((c) => c.substatesGet(id, { + version: version ?? null, + local_search_only: local_search_only ?? false, +})); export const listSubstates = ( - request: ListSubstatesRequest -): Promise => jsonRpc("list_substates", request); -export const getConnections = (): Promise => - jsonRpc("get_connections"); -// export const getNonFungibleCount = ( -// request: GetNonFungibleCountRequest -// ): Promise => -// jsonRpc('get_non_fungible_count', request); + request: ListSubstatesRequest, +): Promise => client().then((c) => c.listSubstates(request)); export const getNonFungibles = ( - request: GetNonFungiblesRequest -): Promise => jsonRpc("get_non_fungibles", request); -export const submitTransaction = ( - request: GetNonFungiblesRequest -): Promise => - jsonRpc("submit_transaction", request); + request: GetNonFungiblesRequest, +): Promise => + client().then((c) => c.getNonFungibles(request)); + export const getTransactionResult = ( - request: IndexerGetTransactionResultRequest + request: IndexerGetTransactionResultRequest, ): Promise => - jsonRpc("get_transaction_result", request); + client().then((c) => c.getTransactionResult(request.transaction_id)); -export const getEpochManagerStats = (): Promise => - jsonRpc("get_epoch_manager_stats"); export const listRecentTransactions = ( - request: ListRecentTransactionsRequest + request: ListRecentTransactionsRequest, ): Promise => - jsonRpc("list_recent_transactions", request); + client().then((c) => c.listRecentTransactions(request)); diff --git a/applications/tari_swarm_daemon/src/logger.rs b/applications/tari_swarm_daemon/src/logger.rs index a5ac0619ab..e137fd7482 100644 --- a/applications/tari_swarm_daemon/src/logger.rs +++ b/applications/tari_swarm_daemon/src/logger.rs @@ -7,7 +7,15 @@ use fern::{DateBased, FormatCallback}; pub fn init_logger(log_to_file: Option) -> Result<(), log::SetLoggerError> { fn should_skip(target: &str) -> bool { - const SKIP: &[&str] = &["hyper", "h2", "tower", "hyper_util", "tokio", "tokio_util"]; + const SKIP: &[&str] = &[ + "hyper", + "h2", + "tower", + "hyper_util", + "tokio", + "tokio_util", + "cranelift_codegen", + ]; target.is_empty() || SKIP.iter().any(|s| target.starts_with(s)) } diff --git a/applications/tari_swarm_daemon/src/process_definitions/context.rs b/applications/tari_swarm_daemon/src/process_definitions/context.rs index 645b4c0b8f..50c1b5704d 100644 --- a/applications/tari_swarm_daemon/src/process_definitions/context.rs +++ b/applications/tari_swarm_daemon/src/process_definitions/context.rs @@ -106,6 +106,29 @@ impl<'a> ProcessContext<'a> { } } + pub fn get_public_api_url(&self) -> Url { + match self.settings.get("public_api_url") { + Some(url) => url.parse().expect("Invalid API URL"), + None => { + let public_ip = self + .settings + .get("public_ip") + .map(|s| { + if s == "127.0.0.1" { + return "localhost"; + } + s.as_str() + }) + .unwrap_or("localhost"); + let port = self + .port_allocator + .get("api") + .expect("API port must be allocated before calling get_public_api_url"); + format!("http://{public_ip}:{port}").parse().expect("Invalid API URL") + }, + } + } + pub fn get_public_graphql_url(&self) -> Url { match self.settings.get("public_graphql_url") { Some(url) => url.parse().expect("Invalid GraphQL URL"), diff --git a/applications/tari_swarm_daemon/src/process_definitions/indexer.rs b/applications/tari_swarm_daemon/src/process_definitions/indexer.rs index 0cbf80811e..8904d3133f 100644 --- a/applications/tari_swarm_daemon/src/process_definitions/indexer.rs +++ b/applications/tari_swarm_daemon/src/process_definitions/indexer.rs @@ -22,14 +22,14 @@ impl Indexer { impl ProcessDefinition for Indexer { async fn get_command(&self, mut context: ProcessContext<'_>) -> anyhow::Result { let mut command = Command::new(context.bin()); - let jrpc_port = context.get_free_port("jrpc").await?; + let api_port = context.get_free_port("api").await?; let graphql_port = context.get_free_port("graphql").await?; let web_ui_port = context.get_free_port("web").await?; let listen_ip = context.listen_ip(); - let json_rpc_public_url = context.get_public_json_rpc_url(); + let api_public_url = context.get_public_api_url(); let graphql_public_url = context.get_public_graphql_url(); - let json_rpc_listener_address = format!("{listen_ip}:{jrpc_port}"); + let api_listener_address = format!("{listen_ip}:{api_port}"); let graphql_listener_address = format!("{listen_ip}:{graphql_port}"); let web_ui_listener_address = format!("{listen_ip}:{web_ui_port}"); @@ -54,10 +54,10 @@ impl ProcessDefinition for Indexer { .arg(format!( "-pepoch_oracle.base_layer.base_node_grpc_url={base_node_grpc_url}" )) - .arg(format!("-pindexer.json_rpc_address={json_rpc_listener_address}")) + .arg(format!("-pindexer.api_listen_address={api_listener_address}")) .arg(format!("-pindexer.graphql_address={graphql_listener_address}")) .arg(format!("-pindexer.web_ui_address={web_ui_listener_address}")) - .arg(format!("-pindexer.web_ui_public_json_rpc_url={json_rpc_public_url}")) + .arg(format!("-pindexer.web_ui_public_api_url={api_public_url}")) .arg(format!("-pindexer.web_ui_public_graphql_url={graphql_public_url}")) .arg("-pepoch_oracle.base_layer.scanning_interval=1"); diff --git a/applications/tari_swarm_daemon/src/process_definitions/wallet_daemon.rs b/applications/tari_swarm_daemon/src/process_definitions/wallet_daemon.rs index c0e9f7effc..f47993a42f 100644 --- a/applications/tari_swarm_daemon/src/process_definitions/wallet_daemon.rs +++ b/applications/tari_swarm_daemon/src/process_definitions/wallet_daemon.rs @@ -41,12 +41,12 @@ impl ProcessDefinition for WalletDaemon { .next() .ok_or_else(|| anyhow!("Indexer should be started before wallet daemon"))?; let indexer_url = format!( - "http://127.0.0.1:{}/json_rpc", + "http://127.0.0.1:{}", indexer .instance() .allocated_ports() - .get("jrpc") - .ok_or_else(|| anyhow!("Indexer jrpc port not found"))? + .get("api") + .ok_or_else(|| anyhow!("Indexer api port not found"))? ); let auth = context diff --git a/applications/tari_swarm_daemon/src/process_manager/handle.rs b/applications/tari_swarm_daemon/src/process_manager/handle.rs index f26c1ffec4..93e09821c9 100644 --- a/applications/tari_swarm_daemon/src/process_manager/handle.rs +++ b/applications/tari_swarm_daemon/src/process_manager/handle.rs @@ -141,7 +141,32 @@ impl InstanceInfo { let web_port = self.ports.get("jrpc").expect("jrpc port not found"); format!("http://{public_ip}:{web_port}/json_rpc") .parse() - .expect("Invalid web URL") + .expect("Invalid JSON-RPC URL") + }, + } + } + + pub fn get_public_api_url(&self) -> Url { + match self.settings.get("public_api_url") { + Some(url) => url + .parse() + .expect("Invalid API URL. Please check `public_api_url` in your config."), + None => { + let public_ip = self + .settings + .get("public_ip") + .map(|s| { + // Required for webauthn + if s == "127.0.0.1" { + return "localhost"; + } + s.as_str() + }) + .unwrap_or("localhost"); + let web_port = self.ports.get("api").expect("api port not found"); + format!("http://{public_ip}:{web_port}") + .parse() + .expect("Invalid api URL") }, } } diff --git a/applications/tari_swarm_daemon/src/webserver/rpc/indexers.rs b/applications/tari_swarm_daemon/src/webserver/rpc/indexers.rs index 23ab82a461..4f565edc47 100644 --- a/applications/tari_swarm_daemon/src/webserver/rpc/indexers.rs +++ b/applications/tari_swarm_daemon/src/webserver/rpc/indexers.rs @@ -21,7 +21,7 @@ pub struct IndexerInfo { pub instance_id: InstanceId, pub name: String, pub web: Url, - pub jrpc: Url, + pub api_url: Url, pub graphql: Url, pub is_running: bool, } @@ -32,7 +32,7 @@ pub async fn list(context: &HandlerContext, _req: ListIndexersRequest) -> Result let nodes = instances .into_iter() .map(|instance| { - let jrpc = instance.get_public_json_rpc_url(); + let api_url = instance.get_public_api_url(); let graphql: Url = instance.get_public_graphql_url(); let web = instance.get_public_web_url(); @@ -40,7 +40,7 @@ pub async fn list(context: &HandlerContext, _req: ListIndexersRequest) -> Result instance_id: instance.id, name: instance.name, web, - jrpc, + api_url, graphql, is_running: instance.is_running, }) diff --git a/applications/tari_swarm_daemon/webui/src/routes/Main.tsx b/applications/tari_swarm_daemon/webui/src/routes/Main.tsx index dba0cc1f59..06c0f83527 100644 --- a/applications/tari_swarm_daemon/webui/src/routes/Main.tsx +++ b/applications/tari_swarm_daemon/webui/src/routes/Main.tsx @@ -288,6 +288,12 @@ function ShowInfo(params: any) { {node.jrpc} ); + const apiInfo = node?.api_url && ( +
+ API + {node.api_url} +
+ ); const graphQLInfo = node?.graphql && (
GraphQL @@ -353,6 +359,7 @@ function ShowInfo(params: any) { {nameInfo} {httpInfo} {jrpcInfo} + {apiInfo} {graphQLInfo} {grpcInfo} {showLogs && logInfo} diff --git a/applications/tari_validator_node/web_ui/src/utils/json_rpc.tsx b/applications/tari_validator_node/web_ui/src/utils/json_rpc.tsx index 2a1f3a655f..c5867ce57b 100644 --- a/applications/tari_validator_node/web_ui/src/utils/json_rpc.tsx +++ b/applications/tari_validator_node/web_ui/src/utils/json_rpc.tsx @@ -144,5 +144,4 @@ export const getNetworkCommittees = (): Promise => // Comms export const addPeer = (request: VNAddPeerRequest) => jsonRpc("add_peer", request); -export const getCommsStats = (): Promise => jsonRpc("get_comms_stats"); export const getConnections = (): Promise => jsonRpc("get_connections"); diff --git a/applications/tari_walletd/Cargo.toml b/applications/tari_walletd/Cargo.toml index a1b2647890..2266467795 100644 --- a/applications/tari_walletd/Cargo.toml +++ b/applications/tari_walletd/Cargo.toml @@ -16,7 +16,7 @@ tari_ootle_app_utilities = { workspace = true } tari_shutdown = { workspace = true } tari_ootle_wallet_crypto = { workspace = true } tari_ootle_wallet_sdk = { workspace = true } -tari_ootle_wallet_sdk_services = { workspace = true, features = ["indexer_jrpc"] } +tari_ootle_wallet_sdk_services = { workspace = true, features = ["indexer_client"] } tari_ootle_wallet_storage_sqlite = { workspace = true } tari_transaction = { workspace = true } tari_ootle_common_types = { workspace = true } diff --git a/applications/tari_walletd/src/cli.rs b/applications/tari_walletd/src/cli.rs index 2ea106a73d..e42537cc22 100644 --- a/applications/tari_walletd/src/cli.rs +++ b/applications/tari_walletd/src/cli.rs @@ -52,8 +52,8 @@ pub struct Cli { #[clap(long, env = "SIGNALING_SERVER_ADDRESS")] pub signaling_server_address: Option, #[clap(long, short = 'i', alias = "indexer-url")] - /// Indexer JSON-RPC url override - pub indexer_json_rpc_url: Option, + /// Indexer API url override + pub indexer_api_url: Option, #[clap(flatten)] pub wallet_restore: WalletRestoreArgs, /// The OS keyring is used to store and retrieve a randomly generated password. This is used for wallet encryption. @@ -106,10 +106,10 @@ impl ConfigOverrideProvider for Cli { signaling_server_address.to_string(), )); } - if let Some(ref indexer_json_rpc_url) = self.indexer_json_rpc_url { + if let Some(ref indexer_api_url) = self.indexer_api_url { overrides.push(( - "ootle_wallet_daemon.indexer_json_rpc_url".to_string(), - indexer_json_rpc_url.to_string(), + "ootle_wallet_daemon.indexer_api_url".to_string(), + indexer_api_url.to_string(), )); } if let Some(ref file) = self.value_lookup_table_file { diff --git a/applications/tari_walletd/src/config.rs b/applications/tari_walletd/src/config.rs index 23a4d56bed..63f8d72eb9 100644 --- a/applications/tari_walletd/src/config.rs +++ b/applications/tari_walletd/src/config.rs @@ -59,8 +59,8 @@ pub struct WalletDaemonConfig { pub web_ui_public_json_rpc_url: Option, /// The signaling server address for the webrtc pub signaling_server_address: Option, - /// The indexer JSON-RPC url - pub indexer_json_rpc_url: Url, + /// The indexer API url + pub indexer_api_url: Url, /// Expiration duration of the JWT token #[serde(with = "humantime_serde")] #[serde(default = "return_default_jwt_expiry")] @@ -96,9 +96,9 @@ impl Default for WalletDaemonConfig { json_rpc_address: Some(SocketAddr::from(([127u8, 0, 0, 1], 9000))), web_ui_public_json_rpc_url: None, signaling_server_address: Some(SocketAddr::from(([127u8, 0, 0, 1], 9100))), - indexer_json_rpc_url: "http://127.0.0.1:18300/json_rpc" + indexer_api_url: "http://127.0.0.1:18300" .parse() - .expect("failed to parse default indexer_json_rpc_url"), + .expect("failed to parse default indexer_api_url"), jwt_expiry: return_default_jwt_expiry(), jwt_secret_key: create_secret_password(), web_ui_address: Some("127.0.0.1:5100".parse().unwrap()), diff --git a/applications/tari_walletd/src/handlers/accounts.rs b/applications/tari_walletd/src/handlers/accounts.rs index b893dbe15b..e36ab8bbef 100644 --- a/applications/tari_walletd/src/handlers/accounts.rs +++ b/applications/tari_walletd/src/handlers/accounts.rs @@ -959,22 +959,76 @@ pub async fn handle_stealth_transfer( task::spawn(async move { let transfer = sdk.stealth_transfer_api().transfer(owner_account, params).await?; + // TODO: if submitting fails we need to unlock the inputs again if req.dry_run { let transaction_id = transfer.transaction.calculate_id(); - transaction_service + let result = transaction_service .submit_dry_run_transaction(transfer.transaction) - .await?; - return Ok(StealthTransferResponse { transaction_id }); - } - - let tx_id = transaction_service.submit_transaction(transfer.transaction).await?; + .await; + return match result { + Ok(_) => Ok(StealthTransferResponse { transaction_id }), + Err(e) => { + if let Err(err) = sdk + .stealth_outputs_api() + .release_locked_outputs(transfer.transaction_lock_id) + { + error!( + target: LOG_TARGET, + "Failed to release locked outputs after dry run failure: {}", + err + ); + } + if let Err(err) = sdk + .stealth_outputs_api() + .release_revealed_funds(transfer.transaction_lock_id) + { + error!( + target: LOG_TARGET, + "Failed to release revealed funds after dry run failure: {}", + err + ); + } - notifier.notify(TransactionSubmittedEvent { - transaction_id: tx_id, - new_account: None, - }); + Err(anyhow::anyhow!("Dry run transaction failed: {}", e)) + }, + }; + } - Ok(StealthTransferResponse { transaction_id: tx_id }) + let result = transaction_service.submit_transaction(transfer.transaction).await; + match result { + Ok(tx_id) => { + notifier.notify(TransactionSubmittedEvent { + transaction_id: tx_id, + new_account: None, + }); + + Ok(StealthTransferResponse { transaction_id: tx_id }) + }, + Err(e) => { + if let Err(err) = sdk + .stealth_outputs_api() + .release_locked_outputs(transfer.transaction_lock_id) + { + error!( + target: LOG_TARGET, + "Failed to release locked outputs after submission failure: {}", + err + ); + } + if let Err(err) = sdk + .stealth_outputs_api() + .release_revealed_funds(transfer.transaction_lock_id) + { + error!( + target: LOG_TARGET, + "Failed to release revealed funds after submission failure: {}", + err + ); + } + + Err(anyhow::anyhow!("Transaction submission failed: {}", e)) + }, + } }) .await? } diff --git a/applications/tari_walletd/src/handlers/context.rs b/applications/tari_walletd/src/handlers/context.rs index e5fc3e2f85..c7875dbfbc 100644 --- a/applications/tari_walletd/src/handlers/context.rs +++ b/applications/tari_walletd/src/handlers/context.rs @@ -2,11 +2,9 @@ // SPDX-License-Identifier: BSD-3-Clause use axum_extra::headers::authorization::Bearer; -use tari_ootle_wallet_sdk::WalletSdk; use tari_ootle_wallet_sdk_services::{ account_monitor::AccountMonitorHandle, events::WalletEvent, - indexer_jrpc::IndexerJsonRpcNetworkInterface, notify::Notify, transaction_service::TransactionServiceHandle, }; @@ -23,11 +21,12 @@ use crate::{ WalletAuthenticator, }, services::WebauthnService, + WalletSdk, }; #[derive(Debug, Clone)] pub struct HandlerContext { - wallet_sdk: WalletSdk, + wallet_sdk: WalletSdk, notifier: Notify, transaction_service: TransactionServiceHandle, account_monitor: AccountMonitorHandle, @@ -38,7 +37,7 @@ pub struct HandlerContext { impl HandlerContext { pub fn new( - wallet_sdk: WalletSdk, + wallet_sdk: WalletSdk, notifier: Notify, transaction_service: TransactionServiceHandle, account_monitor: AccountMonitorHandle, @@ -61,7 +60,7 @@ impl HandlerContext { &self.notifier } - pub fn wallet_sdk(&self) -> &WalletSdk { + pub fn wallet_sdk(&self) -> &WalletSdk { &self.wallet_sdk } diff --git a/applications/tari_walletd/src/handlers/helpers.rs b/applications/tari_walletd/src/handlers/helpers.rs index b6928b49e6..0ee7a49541 100644 --- a/applications/tari_walletd/src/handlers/helpers.rs +++ b/applications/tari_walletd/src/handlers/helpers.rs @@ -15,11 +15,7 @@ use tari_ootle_wallet_sdk::{ storage::WalletStore, WalletSdk, }; -use tari_ootle_wallet_sdk_services::{ - events::{TransactionFinalizedEvent, WalletEvent}, - indexer_jrpc::IndexerJsonRpcNetworkInterface, -}; -use tari_ootle_wallet_storage_sqlite::SqliteWalletStore; +use tari_ootle_wallet_sdk_services::events::{TransactionFinalizedEvent, WalletEvent}; use tari_template_builtin::ACCOUNT_TEMPLATE_ADDRESS; use tari_template_lib::models::ComponentAddress; use tari_transaction::TransactionId; @@ -93,10 +89,15 @@ pub async fn wait_for_result_and_account( } } -pub fn get_account_with_inputs( +pub fn get_account_with_inputs( account: Option<&ComponentAddressOrName>, - sdk: &WalletSdk, -) -> Result<(AccountWithAddress, HashSet), anyhow::Error> { + sdk: &WalletSdk, +) -> Result<(AccountWithAddress, HashSet), anyhow::Error> +where + TStore: WalletStore, + TNetworkInterface: WalletNetworkInterface, + TNetworkInterface::Error: IsNotFoundError + StatusResponseError, +{ let account = get_account_or_default(account, &sdk.accounts_api())?; let inputs = if account.is_confirmed_on_chain() { // Add all versioned account child addresses as inputs diff --git a/applications/tari_walletd/src/lib.rs b/applications/tari_walletd/src/lib.rs index 656760e1da..27bc2b90bd 100644 --- a/applications/tari_walletd/src/lib.rs +++ b/applications/tari_walletd/src/lib.rs @@ -40,12 +40,12 @@ use tari_ootle_wallet_sdk::{ key_manager::KeyBranch, }, cipher_seed::CipherSeedRestore, - WalletSdk, + WalletSdk as Sdk, WalletSdkConfig, }; use tari_ootle_wallet_sdk_services::{ account_recovery::AccountRecoveryService, - indexer_jrpc::IndexerJsonRpcNetworkInterface, + indexer_rest_api::IndexerRestApiNetworkInterface, notify::Notify, }; use tari_ootle_wallet_storage_sqlite::SqliteWalletStore; @@ -64,6 +64,8 @@ const DEFAULT_FEE: u64 = 1500; // TODO: must match the global network value. All testnets currently have 256 pre-shards. const NUM_PRESHARDS: NumPreshards = NumPreshards::current(); +pub type WalletSdk = Sdk; + pub async fn run_tari_ootle_walletd( config: ApplicationConfig, seed_words: Option<&SeedWords>, @@ -184,21 +186,18 @@ pub fn init_wallet_store(config: &ApplicationConfig) -> anyhow::Result anyhow::Result> { +pub fn initialize_wallet_sdk(config: &ApplicationConfig, store: SqliteWalletStore) -> anyhow::Result { let sdk_config = WalletSdkConfig { network: config.ootle_wallet_daemon.network, override_keyring_password: config.ootle_wallet_daemon.override_keyring_password.clone(), }; let config_api = ConfigApi::new(&store); - let indexer_jrpc_endpoint = if let Some(indexer_url) = config_api.get(ConfigKey::IndexerUrl).optional()? { + let indexer_endpoint = if let Some(indexer_url) = config_api.get(ConfigKey::IndexerUrl).optional()? { indexer_url } else { - config.ootle_wallet_daemon.indexer_json_rpc_url.clone() + config.ootle_wallet_daemon.indexer_api_url.clone() }; - let indexer = IndexerJsonRpcNetworkInterface::new(indexer_jrpc_endpoint); + let indexer = IndexerRestApiNetworkInterface::new(indexer_endpoint); let sdk = WalletSdk::initialize(store, indexer, sdk_config)?; Ok(sdk) } diff --git a/applications/tari_walletd/src/main.rs b/applications/tari_walletd/src/main.rs index 4d662af8de..e7c984062f 100644 --- a/applications/tari_walletd/src/main.rs +++ b/applications/tari_walletd/src/main.rs @@ -172,7 +172,7 @@ async fn run(cli: Cli, config: ApplicationConfig) -> Result<(), anyhow::Error> { target: LOG_TARGET, "🟢 Starting wallet on {} connected to indexer {}", config.ootle_wallet_daemon.network, - config.ootle_wallet_daemon.indexer_json_rpc_url + config.ootle_wallet_daemon.indexer_api_url ); run_tari_ootle_walletd(config, cli.wallet_restore.seed_words.as_ref(), shutdown_signal).await diff --git a/bindings/package.json b/bindings/package.json index b39714e081..0a95ed810a 100644 --- a/bindings/package.json +++ b/bindings/package.json @@ -1,6 +1,6 @@ { "name": "@tari-project/typescript-bindings", - "version": "1.17.2", + "version": "1.18.0", "description": "TypeScript types synchronized to the Tari Ootle Rust codebase", "homepage": "https://github.com/tari-project/tari-ootle#readme", "bugs": { diff --git a/bindings/src/tari-indexer-client.ts b/bindings/src/tari-indexer-client.ts index e39da20c24..d0487ad191 100644 --- a/bindings/src/tari-indexer-client.ts +++ b/bindings/src/tari-indexer-client.ts @@ -21,6 +21,7 @@ export * from "./types/tari-indexer-client/IndexerConnection"; export * from "./types/tari-indexer-client/GetNonFungiblesRequest"; export * from "./types/tari-indexer-client/ListRecentTransactionsResponse"; export * from "./types/tari-indexer-client/IndexerAddPeerRequest"; +export * from "./types/tari-indexer-client/GetNetworkSyncStateResponse"; export * from "./types/tari-indexer-client/GetNonFungiblesResponse"; export * from "./types/tari-indexer-client/GetTemplateDefinitionResponse"; export * from "./types/tari-indexer-client/IndexerGetConnectionsResponse"; diff --git a/bindings/src/types/tari-indexer-client/GetNetworkSyncStateResponse.ts b/bindings/src/types/tari-indexer-client/GetNetworkSyncStateResponse.ts new file mode 100644 index 0000000000..5386b41845 --- /dev/null +++ b/bindings/src/types/tari-indexer-client/GetNetworkSyncStateResponse.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { NetworkDescription } from "./NetworkDescription"; +import type { SyncProgress } from "./SyncProgress"; + +export type GetNetworkSyncStateResponse = { network_desc: NetworkDescription; sync_progress: SyncProgress | null }; diff --git a/bindings/src/types/tari-indexer-client/GetUtxoUpdatesRequest.ts b/bindings/src/types/tari-indexer-client/GetUtxoUpdatesRequest.ts index 63a5e39cae..12cb894a25 100644 --- a/bindings/src/types/tari-indexer-client/GetUtxoUpdatesRequest.ts +++ b/bindings/src/types/tari-indexer-client/GetUtxoUpdatesRequest.ts @@ -6,5 +6,6 @@ import type { StateVersion } from "../StateVersion"; export type GetUtxoUpdatesRequest = { shard_state_versions: Array<[Shard, StateVersion]>; resource_address: ResourceAddress; + unspent_only?: boolean; per_shard_limit: number; }; diff --git a/bindings/src/types/tari-indexer-client/IndexerGetEpochManagerStatsResponse.ts b/bindings/src/types/tari-indexer-client/IndexerGetEpochManagerStatsResponse.ts index 0e5f7aa3b7..07cdb21447 100644 --- a/bindings/src/types/tari-indexer-client/IndexerGetEpochManagerStatsResponse.ts +++ b/bindings/src/types/tari-indexer-client/IndexerGetEpochManagerStatsResponse.ts @@ -4,5 +4,5 @@ import type { Epoch } from "../Epoch"; export type IndexerGetEpochManagerStatsResponse = { current_epoch: Epoch; current_block_height: number; - current_block_hash: string; + current_block_hash: number[]; }; diff --git a/bindings/src/types/tari-indexer-client/IndexerGetIdentityResponse.ts b/bindings/src/types/tari-indexer-client/IndexerGetIdentityResponse.ts index 99db681649..cd3d5752fb 100644 --- a/bindings/src/types/tari-indexer-client/IndexerGetIdentityResponse.ts +++ b/bindings/src/types/tari-indexer-client/IndexerGetIdentityResponse.ts @@ -4,5 +4,5 @@ import type { RistrettoPublicKeyBytes } from "../RistrettoPublicKeyBytes"; export type IndexerGetIdentityResponse = { peer_id: string; public_key: RistrettoPublicKeyBytes; - public_addresses: Array; + public_addresses: string[]; }; diff --git a/bindings/src/types/tari-indexer-client/IndexerGetSubstateRequest.ts b/bindings/src/types/tari-indexer-client/IndexerGetSubstateRequest.ts index ecdf692558..ca60508cfb 100644 --- a/bindings/src/types/tari-indexer-client/IndexerGetSubstateRequest.ts +++ b/bindings/src/types/tari-indexer-client/IndexerGetSubstateRequest.ts @@ -1,3 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type IndexerGetSubstateRequest = { address: string; version: number | null; local_search_only: boolean }; +export type IndexerGetSubstateRequest = { version: number | null; local_search_only: boolean }; diff --git a/bindings/src/types/tari-indexer-client/IndexerGetSubstateResponse.ts b/bindings/src/types/tari-indexer-client/IndexerGetSubstateResponse.ts index f01e5ea2ff..974d3ba957 100644 --- a/bindings/src/types/tari-indexer-client/IndexerGetSubstateResponse.ts +++ b/bindings/src/types/tari-indexer-client/IndexerGetSubstateResponse.ts @@ -1,5 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { SubstateId } from "../SubstateId"; import type { SubstateValue } from "../SubstateValue"; -export type IndexerGetSubstateResponse = { address: SubstateId; version: number; substate: SubstateValue }; +export type IndexerGetSubstateResponse = { version: number; substate: SubstateValue }; diff --git a/bindings/src/types/tari-indexer-client/InspectSubstateResponse.ts b/bindings/src/types/tari-indexer-client/InspectSubstateResponse.ts index cb527245e5..0e55d6eabb 100644 --- a/bindings/src/types/tari-indexer-client/InspectSubstateResponse.ts +++ b/bindings/src/types/tari-indexer-client/InspectSubstateResponse.ts @@ -1,5 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { SubstateId } from "../SubstateId"; import type { SubstateValue } from "../SubstateValue"; -export type InspectSubstateResponse = { address: SubstateId; version: number; substate: SubstateValue }; +export type InspectSubstateResponse = { version: number; substate: SubstateValue }; diff --git a/bindings/src/types/tari-indexer-client/ListTemplatesRequest.ts b/bindings/src/types/tari-indexer-client/ListTemplatesRequest.ts index a4335d711a..26cc73caad 100644 --- a/bindings/src/types/tari-indexer-client/ListTemplatesRequest.ts +++ b/bindings/src/types/tari-indexer-client/ListTemplatesRequest.ts @@ -1,3 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type ListTemplatesRequest = { limit: number }; +export type ListTemplatesRequest = { limit: number | null }; diff --git a/bindings/src/types/tari-indexer-client/TemplateMetadata.ts b/bindings/src/types/tari-indexer-client/TemplateMetadata.ts index 451fcd3ad1..47638dcf4a 100644 --- a/bindings/src/types/tari-indexer-client/TemplateMetadata.ts +++ b/bindings/src/types/tari-indexer-client/TemplateMetadata.ts @@ -7,5 +7,5 @@ export type TemplateMetadata = { /** * SHA hash of binary */ - binary_sha: string; + binary_sha: number[]; }; diff --git a/clients/javascript/indexer_client/.gitignore b/clients/javascript/indexer_client/.gitignore new file mode 100644 index 0000000000..4730b76927 --- /dev/null +++ b/clients/javascript/indexer_client/.gitignore @@ -0,0 +1,3 @@ +dist/ +node_modules/ +tsconfig.tsbuildinfo diff --git a/clients/javascript/indexer_client/.prettierrc b/clients/javascript/indexer_client/.prettierrc new file mode 100644 index 0000000000..b302ba926d --- /dev/null +++ b/clients/javascript/indexer_client/.prettierrc @@ -0,0 +1,12 @@ +{ + "printWidth": 120, + "tabWidth": 2, + "useTabs": false, + "semi": true, + "singleQuote": false, + "trailingComma": "all", + "bracketSpacing": true, + "bracketSameLine": false, + "arrowParens": "always", + "endOfLine": "auto" +} diff --git a/clients/javascript/indexer_client/README.md b/clients/javascript/indexer_client/README.md new file mode 100644 index 0000000000..bdbf0d4494 --- /dev/null +++ b/clients/javascript/indexer_client/README.md @@ -0,0 +1,5 @@ +# Tari Ootle Indexer API Client + +## Overview + +Client library for interacting with the Tari Ootle Indexer API. diff --git a/clients/javascript/indexer_client/package.json b/clients/javascript/indexer_client/package.json new file mode 100644 index 0000000000..b700edb255 --- /dev/null +++ b/clients/javascript/indexer_client/package.json @@ -0,0 +1,35 @@ +{ + "name": "@tari-project/indexer-client", + "version": "1.0.0", + "description": "Tari indexer REST API client library", + "homepage": "https://github.com/tari-project/tari-ootle#readme", + "bugs": { + "url": "https://github.com/tari-project/tari-ootle/issues" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tari-project/tari-ootle.git" + }, + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc --project tsconfig.prod.json", + "test": "echo \"Error: no test specified\" && exit 1", + "clean-dist": "rm -rf dist && rm -f tsconfig.tsbuildinfo" + }, + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist" + ], + "author": "", + "license": "ISC", + "dependencies": { + "@tari-project/typescript-bindings": "workspace:^" + }, + "devDependencies": { + "@types/node": "catalog:", + "typescript": "catalog:" + } +} \ No newline at end of file diff --git a/clients/javascript/indexer_client/src/index.ts b/clients/javascript/indexer_client/src/index.ts new file mode 100644 index 0000000000..9ee10ebb60 --- /dev/null +++ b/clients/javascript/indexer_client/src/index.ts @@ -0,0 +1,122 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + +import type { + GetEpochManagerStatsResponse, + GetNetworkSyncStateResponse, GetNonFungiblesRequest, GetNonFungiblesResponse, + GetRecentTransactionsRequest, + GetRecentTransactionsResponse, + GetSubstatesRequest, + GetSubstatesResponse, + IndexerAddPeerRequest, + IndexerAddPeerResponse, + IndexerGetConnectionsResponse, + IndexerGetIdentityResponse, + IndexerGetSubstateRequest, + IndexerGetSubstateResponse, + IndexerGetTransactionResultResponse, + IndexerReadyResponse, ListRecentTransactionsRequest, ListRecentTransactionsResponse, + ListSubstatesRequest, + ListSubstatesResponse, ListTemplatesResponse, + rejectReasonToString, + stringToSubstateId, + SubstateId, + substateIdToString, + TemplatesGetResponse, + TemplatesListAuthoredRequest, + TemplatesListAuthoredResponse, + TransactionId, + TransactionSubmitRequest, + TransactionSubmitResponse, +} from "@tari-project/typescript-bindings"; +import { FetchTransport, HttpTransport } from "./transports"; + +export * as transports from "./transports"; + +export { substateIdToString, stringToSubstateId, rejectReasonToString }; + +export class IndexerClient { + private transport: HttpTransport; + + constructor(transport: HttpTransport) { + this.transport = transport; + } + + public static new(transport: HttpTransport): IndexerClient { + return new IndexerClient(transport); + } + + public static usingFetchTransport(url: string): IndexerClient { + return IndexerClient.new(FetchTransport.new(url)); + } + + getTransport() { + return this.transport; + } + + public identityGet(): Promise { + return this.transport.sendGet(`identity`, {}); + } + + public waitUntilReady(): Promise { + return this.transport.sendGet(`wait-until-ready`, {}); + } + + public epochManagerStats(): Promise { + return this.transport.sendGet(`epoch-manager/stats`, {}); + } + + public networkStats(): Promise { + return this.transport.sendGet(`network/stats`, {}); + } + + public addConnection(req: IndexerAddPeerRequest): Promise { + return this.transport.sendPost(`network/connections`, req); + } + + public getConnections(): Promise { + return this.transport.sendGet(`network/connections`, {}); + } + + public getNonFungibles(params: GetNonFungiblesRequest): Promise { + return this.transport.sendGet(`non-fungibles`, params); + } + + public substatesGet(id: SubstateId, params: IndexerGetSubstateRequest): Promise { + return this.transport.sendGet(`substates/${encodeURIComponent(id)}`, params); + } + + public listSubstates(params: ListSubstatesRequest): Promise { + return this.transport.sendGet(`substates`, params); + } + + public fetchSubstates(params: GetSubstatesRequest): Promise { + return this.transport.sendPost(`substates/fetch`, params); + } + + public submitTransaction(params: TransactionSubmitRequest): Promise { + return this.transport.sendPost(`transactions`, params); + } + + public getTransactionResult(transaction_id: TransactionId): Promise { + return this.transport.sendGet(`transactions/${encodeURIComponent(transaction_id)}/result`, {}); + } + + public listRecentTransactions(params: ListRecentTransactionsRequest): Promise { + return this.transport.sendGet(`transactions/recent`, params); + } + + public templatesGet(template_address: string): Promise { + return this.transport.sendGet(`templates/${encodeURIComponent(template_address)}`, {}); + } + + public templatesList(limit: number = 0): Promise { + return this.transport.sendGet(`templates`, { limit }); + } + + public templatesListAuthored(params: TemplatesListAuthoredRequest): Promise { + return this.transport.sendPost(`templates`, params); + } +} diff --git a/clients/javascript/indexer_client/src/transports/fetch.ts b/clients/javascript/indexer_client/src/transports/fetch.ts new file mode 100644 index 0000000000..f88429df50 --- /dev/null +++ b/clients/javascript/indexer_client/src/transports/fetch.ts @@ -0,0 +1,102 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + +import { HttpTransport, TransportOptions } from "./index"; + +export default class FetchTransport implements HttpTransport { + private url: string; + + constructor(url: string) { + this.url = url; + } + + static new(url: string) { + if (!url.startsWith("http://") && !url.startsWith("https://")) { + throw new Error("URL must start with http:// or https://"); + } + if (url.endsWith("/")) { + url = url.slice(0, -1); + } + return new FetchTransport(url); + } + + sendGet(path: string, params: any, options?: TransportOptions): Promise { + return this.sendRequest(path, { method: "GET", params }, options); + } + + sendHead(path: string, params: any, options?: TransportOptions): Promise { + return this.sendRequest(path, { method: "HEAD", params }, options); + } + + sendPost(path: string, body: any, options?: TransportOptions): Promise { + return this.sendRequest(path, { method: "POST", body }, options); + } + + sendPut(path: string, body: any, options?: TransportOptions): Promise { + return this.sendRequest(path, { method: "PUT", body }, options); + } + + sendDelete(path: string, body: any, options?: TransportOptions): Promise { + return this.sendRequest(path, { method: "DELETE", body }, options); + } + + async sendRequest(path: string, request: RequestSpec, options?: TransportOptions): Promise { + let controller = new AbortController(); + let signal = controller.signal; + + const timeoutId = options?.timeout_millis + ? setTimeout(() => { + controller.abort("Timeout"); + }, options.timeout_millis) + : null; + + if (path.startsWith("/")) { + path = path.slice(1); + } + + let query = null; + if (typeof request.params === "object" && request.params !== null) { + const urlParams = new URLSearchParams(); + for (const [key, value] of Object.entries(request.params)) { + if (value !== undefined && value !== null) { + urlParams.append(key, value.toString()); + } + } + query = urlParams; + } + if (typeof request.body === "object" && request.body !== null) { + request.body = JSON.stringify(request.body); + } + + let fqPath = `${this.url}/${path}`; + if (query && query.toString().length > 0) { + fqPath += `?${query.toString()}`; + } + + const response = await fetch(fqPath, { + signal, + ...request, + }); + if (timeoutId) { + clearTimeout(timeoutId); + } + if (!response.ok) { + const text = await response.text(); + throw new Error(`HTTP ${response.status}: ${response.statusText}${text ? ` - ${text}` : ""}`); + } + const json = await response.json(); + if (json.error) { + throw new Error(`${json.error.code}: ${json.error.message}`); + } + return json; + } +} + +export interface RequestSpec { + method: "GET" | "HEAD" | "POST" | "PUT" | "DELETE"; + headers?: Record; + params?: any; + body?: any; +} diff --git a/clients/javascript/indexer_client/src/transports/index.ts b/clients/javascript/indexer_client/src/transports/index.ts new file mode 100644 index 0000000000..fa4826b8bf --- /dev/null +++ b/clients/javascript/indexer_client/src/transports/index.ts @@ -0,0 +1,25 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + +import FetchTransport from "./fetch"; + +export { FetchTransport }; + +export interface HttpTransport { + sendGet(path: string, params: any, options?: TransportOptions): Promise; + + sendHead(path: string, params: any, options?: TransportOptions): Promise; + + sendPost(path: string, body: any, options?: TransportOptions): Promise; + + sendPut(path: string, body: any, options?: TransportOptions): Promise; + + sendDelete(path: string, params: Record, options?: TransportOptions): Promise; +} + +export interface TransportOptions { + timeout_millis?: number; +} + diff --git a/clients/javascript/indexer_client/tsconfig.json b/clients/javascript/indexer_client/tsconfig.json new file mode 100644 index 0000000000..2fb9ad6405 --- /dev/null +++ b/clients/javascript/indexer_client/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "module": "ES2020", + "target": "ESNext", + "moduleResolution": "node", + "declaration": true, + "rootDir": "./src", + "outDir": "./dist", + "esModuleInterop": true, + // Required for wallet web ui to build + "composite": true + }, + "include": [ + "src/**/*" + ], + "references": [ + { + "path": "../../../bindings" + } + ] +} diff --git a/clients/javascript/indexer_client/tsconfig.prod.json b/clients/javascript/indexer_client/tsconfig.prod.json new file mode 100644 index 0000000000..5680f40b05 --- /dev/null +++ b/clients/javascript/indexer_client/tsconfig.prod.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "composite": false + } +} diff --git a/clients/javascript/wallet_daemon_client/package.json b/clients/javascript/wallet_daemon_client/package.json index 541cbaefed..6be68d6bea 100644 --- a/clients/javascript/wallet_daemon_client/package.json +++ b/clients/javascript/wallet_daemon_client/package.json @@ -21,7 +21,7 @@ "main": "dist/index.js", "types": "dist/index.d.ts", "files": [ - "/dist" + "dist" ], "author": "", "license": "ISC", diff --git a/clients/javascript/wallet_daemon_client/src/transports/fetch.ts b/clients/javascript/wallet_daemon_client/src/transports/fetch.ts index bda75388ea..fc213b3f6a 100644 --- a/clients/javascript/wallet_daemon_client/src/transports/fetch.ts +++ b/clients/javascript/wallet_daemon_client/src/transports/fetch.ts @@ -29,8 +29,8 @@ export default class FetchRpcTransport implements RpcTransport { const timeoutId = options.timeout_millis ? setTimeout(() => { - controller.abort("Timeout"); - }, options.timeout_millis) + controller.abort("Timeout"); + }, options.timeout_millis) : null; const response = await fetch(this.url, { @@ -42,6 +42,11 @@ export default class FetchRpcTransport implements RpcTransport { if (timeoutId) { clearTimeout(timeoutId); } + + if (!response.ok) { + const text = await response.text(); + throw new Error(`HTTP ${response.status}: ${response.statusText}${text ? ` - ${text}` : ""}`); + } const json = await response.json(); if (json.error) { throw new Error(`${json.error.code}: ${json.error.message}`); diff --git a/clients/tari_indexer_client/Cargo.toml b/clients/tari_indexer_client/Cargo.toml index 53e900b446..7b9f87367c 100644 --- a/clients/tari_indexer_client/Cargo.toml +++ b/clients/tari_indexer_client/Cargo.toml @@ -18,17 +18,26 @@ tari_template_lib_types = { workspace = true } tari_consensus_types = { workspace = true } tari_ootle_wallet_sdk = { workspace = true } -anyhow = { workspace = true, optional = true } +anyhow = { workspace = true } +bytes = { workspace = true } bounded-vec = { workspace = true } -reqwest = { workspace = true, features = ["json"], optional = true } +log = { workspace = true } +reqwest = { workspace = true, features = ["json", "stream"], optional = true } multiaddr = { workspace = true } serde = { workspace = true, default-features = true } serde_json = { workspace = true } -serde_with = { workspace = true } thiserror = { workspace = true, optional = true } ts-rs = { workspace = true, optional = true } +prost = { workspace = true } +serde_urlencoded = { version = "0.7.1", optional = true } +futures = { workspace = true } +utoipa = { workspace = true, optional = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } [features] default = ["client"] ts = ["ts-rs", "tari_template_lib_types/ts", "tari_engine_types/ts", "tari_ootle_common_types/ts", "tari_ootle_wallet_sdk/ts"] -client = ["dep:reqwest", "dep:anyhow", "dep:thiserror"] +client = ["dep:reqwest", "dep:thiserror", "dep:serde_urlencoded"] +utoipa = ["dep:utoipa"] diff --git a/clients/tari_indexer_client/src/conversions.rs b/clients/tari_indexer_client/src/conversions.rs new file mode 100644 index 0000000000..5510e6c5da --- /dev/null +++ b/clients/tari_indexer_client/src/conversions.rs @@ -0,0 +1,25 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use tari_ootle_wallet_sdk::models::WalletUtxoUpdate; + +use crate::protobuf; + +impl From<&WalletUtxoUpdate> for protobuf::WalletUtxoUpdate { + fn from(value: &WalletUtxoUpdate) -> Self { + match value { + WalletUtxoUpdate::Unspent(unspent) => protobuf::WalletUtxoUpdate::Unspent(protobuf::UtxoUnspent { + tag: unspent.tag.value(), + public_nonce: unspent.public_nonce.to_vec(), + }), + WalletUtxoUpdate::Spent(spent) => protobuf::WalletUtxoUpdate::Spent(protobuf::UtxoSpent { + id: spent.id.as_bytes().to_vec(), + version: spent.version, + }), + WalletUtxoUpdate::Burnt(burnt) => protobuf::WalletUtxoUpdate::Burnt(protobuf::UtxoBurnt { + id: burnt.id.as_bytes().to_vec(), + version: burnt.version, + }), + } + } +} diff --git a/clients/tari_indexer_client/src/error.rs b/clients/tari_indexer_client/src/error.rs index 458d0e1731..e73cf3d3ae 100644 --- a/clients/tari_indexer_client/src/error.rs +++ b/clients/tari_indexer_client/src/error.rs @@ -5,7 +5,7 @@ use reqwest::StatusCode; use tari_ootle_common_types::optional::IsNotFoundError; #[derive(Debug, thiserror::Error)] -pub enum IndexerClientError { +pub enum IndexerJrpcClientError { #[error("Failed to deserialize response for method {method}: {source} - response: {response}")] DeserializeResponse { source: serde_json::Error, @@ -25,7 +25,7 @@ pub enum IndexerClientError { InvalidResponse { message: String }, } -impl IsNotFoundError for IndexerClientError { +impl IsNotFoundError for IndexerJrpcClientError { fn is_not_found_error(&self) -> bool { match self { Self::RequestFailedWithStatus { code, .. } => *code == 404, @@ -34,3 +34,37 @@ impl IsNotFoundError for IndexerClientError { } } } + +#[derive(Debug, thiserror::Error)] +pub enum IndexerRestClientError { + #[error("Failed to deserialize response for path {path}: {source}")] + DeserializeResponse { source: anyhow::Error, path: String }, + #[error("Failed to serialize request for path {path}: {source}")] + SerializeRequest { path: String, source: anyhow::Error }, + #[error("Failed to send request: {source}")] + RequestFailed { + #[from] + source: reqwest::Error, + }, + #[error("Server returned an error: {source} details: {}", .details.as_deref().unwrap_or("none"))] + ErrorResponse { + source: reqwest::Error, + details: Option, + }, + #[error("Request failed: code: {code} message: {message}")] + RequestFailedWithStatus { code: i64, message: String }, + #[error("Invalid response: {message}")] + InvalidResponse { message: String }, +} + +impl IsNotFoundError for IndexerRestClientError { + fn is_not_found_error(&self) -> bool { + match self { + Self::RequestFailedWithStatus { code, .. } => *code == 404, + Self::RequestFailed { source, .. } | Self::ErrorResponse { source, .. } => { + source.status().map(|s| s == StatusCode::NOT_FOUND).unwrap_or(false) + }, + _ => false, + } + } +} diff --git a/clients/tari_indexer_client/src/json_rpc_client.rs b/clients/tari_indexer_client/src/json_rpc_client.rs deleted file mode 100644 index 6eea5ffb2a..0000000000 --- a/clients/tari_indexer_client/src/json_rpc_client.rs +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2022. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use reqwest::{header, header::HeaderMap, IntoUrl, Url}; -use serde::{de::DeserializeOwned, Serialize}; -use serde_json as json; -use serde_json::json; - -use crate::{ - error::IndexerClientError, - types::{ - AddPeerRequest, - AddPeerResponse, - GetEpochManagerStatsResponse, - GetNetworkSyncStateResponse, - GetNonFungiblesRequest, - GetNonFungiblesResponse, - GetSubstateRequest, - GetSubstateResponse, - GetSubstatesRequest, - GetSubstatesResponse, - GetTemplateDefinitionRequest, - GetTemplateDefinitionResponse, - GetTransactionResultRequest, - GetTransactionResultResponse, - GetUnspentUtxosRequest, - GetUnspentUtxosResponse, - GetUtxoUpdatesRequest, - GetUtxoUpdatesResponse, - IndexerReadyResponse, - ListRecentTransactionsRequest, - ListRecentTransactionsResponse, - ListSubstatesRequest, - ListSubstatesResponse, - SubmitTransactionRequest, - SubmitTransactionResponse, - }, -}; - -#[derive(Debug, Clone)] -pub struct IndexerJsonRpcClient { - client: reqwest::Client, - endpoint: Url, - request_id: i64, -} - -impl IndexerJsonRpcClient { - pub fn connect(endpoint: T) -> Result { - let client = reqwest::Client::builder() - .default_headers({ - let mut headers = HeaderMap::with_capacity(1); - headers.insert(header::CONTENT_TYPE, "application/json".parse().unwrap()); - headers - }) - .build()?; - - Ok(Self { - client, - endpoint: endpoint.into_url()?, - request_id: 0, - }) - } - - fn next_request_id(&mut self) -> i64 { - self.request_id += 1; - self.request_id - } - - pub async fn add_peer(&mut self, request: AddPeerRequest) -> Result { - self.send_request("add_peer", request).await - } - - pub async fn get_substate(&mut self, req: GetSubstateRequest) -> Result { - self.send_request("get_substate", req).await - } - - pub async fn get_substates( - &mut self, - req: GetSubstatesRequest, - ) -> Result { - self.send_request("get_substates", req).await - } - - pub async fn fetch_substates( - &mut self, - req: GetSubstatesRequest, - ) -> Result { - self.send_request("fetch_substates", req).await - } - - pub async fn list_substates( - &mut self, - req: ListSubstatesRequest, - ) -> Result { - self.send_request("list_substates", req).await - } - - pub async fn submit_transaction( - &mut self, - req: SubmitTransactionRequest, - ) -> Result { - self.send_request("submit_transaction", req).await - } - - pub async fn get_transaction_result( - &mut self, - req: GetTransactionResultRequest, - ) -> Result { - self.send_request("get_transaction_result", req).await - } - - pub async fn get_template_definition( - &mut self, - req: GetTemplateDefinitionRequest, - ) -> Result { - self.send_request("get_template_definition", req).await - } - - pub async fn get_non_fungibles( - &mut self, - req: GetNonFungiblesRequest, - ) -> Result { - self.send_request("get_non_fungibles", req).await - } - - pub async fn get_epoch_manager_stats(&mut self) -> Result { - self.send_request("get_epoch_manager_stats", ()).await - } - - pub async fn list_recent_transactions( - &mut self, - req: ListRecentTransactionsRequest, - ) -> Result { - self.send_request("list_recent_transactions", req).await - } - - pub async fn get_utxo_updates( - &mut self, - req: GetUtxoUpdatesRequest, - ) -> Result { - self.send_request("get_utxo_updates", req).await - } - - pub async fn get_unspent_utxos( - &mut self, - req: GetUnspentUtxosRequest, - ) -> Result { - self.send_request("get_unspent_utxos", req).await - } - - pub async fn get_network_sync_state(&mut self) -> Result { - self.send_request("get_network_sync_state", ()).await - } - - pub async fn wait_until_ready(&mut self) -> Result { - self.send_request("wait_until_ready", ()).await - } - - async fn send_request( - &mut self, - method: &'static str, - params: T, - ) -> Result { - let params = json::to_value(params).map_err(|e| IndexerClientError::SerializeRequest { - source: e, - method: method.to_string(), - })?; - - let request_json = json!( - { - "jsonrpc": "2.0", - "id": self.next_request_id(), - "method": method, - "params": ¶ms, - } - ); - let resp = self - .client - .post(self.endpoint.clone()) - .body(request_json.to_string()) - .send() - .await?; - let json = resp.json().await?; - let resp = jsonrpc_result(&json)?; - // Response might not deserialize to R.... - match serde_json::from_value(resp) { - Ok(r) => Ok(r), - Err(e) => Err(IndexerClientError::DeserializeResponse { - method, - source: e, - response: json, - }), - } - } -} - -fn jsonrpc_result(val: &json::Value) -> Result { - if let Some(err) = val.get("error") { - let code = err.get("code").and_then(|c| c.as_i64()).unwrap_or(-1); - let message = err.get("message").and_then(|m| m.as_str()).unwrap_or("Unknown error"); - return Err(IndexerClientError::RequestFailedWithStatus { - code, - message: message.to_string(), - }); - } - - let result = val.get("result").ok_or_else(|| IndexerClientError::InvalidResponse { - message: "Missing result field".to_string(), - })?; - Ok(result.clone()) -} diff --git a/clients/tari_indexer_client/src/lib.rs b/clients/tari_indexer_client/src/lib.rs index d53a09ae49..a05bf5c75b 100644 --- a/clients/tari_indexer_client/src/lib.rs +++ b/clients/tari_indexer_client/src/lib.rs @@ -20,10 +20,16 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +mod conversions; #[cfg(feature = "client")] pub mod error; #[cfg(feature = "client")] pub mod graphql_client; +pub mod protobuf; +pub mod protobuf_stream; +pub mod rest_api; #[cfg(feature = "client")] -pub mod json_rpc_client; +pub mod rest_api_client; pub mod types; + +pub use prost; diff --git a/clients/tari_indexer_client/src/protobuf.rs b/clients/tari_indexer_client/src/protobuf.rs new file mode 100644 index 0000000000..f3c5800d34 --- /dev/null +++ b/clients/tari_indexer_client/src/protobuf.rs @@ -0,0 +1,62 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +#[derive(::prost::Message, serde::Serialize, serde::Deserialize)] +pub struct UtxoUpdatePayload { + #[prost(message, tag = "1")] + pub sos: Option, + #[prost(oneof = "WalletUtxoUpdate", tags = "2, 3, 4")] + pub update: Option, + #[prost(message, tag = "5")] + pub eos: Option, +} + +#[derive(::prost::Message, serde::Serialize, serde::Deserialize)] +pub struct StartOfShard { + #[prost(uint32, tag = "1")] + pub shard: u32, + #[prost(uint64, tag = "2")] + pub max_state_version: u64, + #[prost(uint32, tag = "3")] + pub num_updates: u32, +} + +#[derive(::prost::Message, serde::Serialize, serde::Deserialize)] +pub struct EndOfShard { + #[prost(uint64, tag = "1")] + pub max_state_version: u64, +} + +#[derive(::prost::Oneof, serde::Serialize, serde::Deserialize)] +pub enum WalletUtxoUpdate { + #[prost(message, tag = "2")] + Unspent(UtxoUnspent), + #[prost(message, tag = "3")] + Spent(UtxoSpent), + #[prost(message, tag = "4")] + Burnt(UtxoBurnt), +} + +#[derive(::prost::Message, serde::Serialize, serde::Deserialize)] +pub struct UtxoUnspent { + #[prost(uint32, tag = "1")] + pub tag: u32, + #[prost(bytes, tag = "2")] + pub public_nonce: Vec, +} + +#[derive(::prost::Message, serde::Serialize, serde::Deserialize)] +pub struct UtxoSpent { + #[prost(bytes, tag = "1")] + pub id: Vec, + #[prost(uint32, tag = "2")] + pub version: u32, +} + +#[derive(::prost::Message, serde::Serialize, serde::Deserialize)] +pub struct UtxoBurnt { + #[prost(bytes, tag = "1")] + pub id: Vec, + #[prost(uint32, tag = "2")] + pub version: u32, +} diff --git a/clients/tari_indexer_client/src/protobuf_stream.rs b/clients/tari_indexer_client/src/protobuf_stream.rs new file mode 100644 index 0000000000..7a10bae6f1 --- /dev/null +++ b/clients/tari_indexer_client/src/protobuf_stream.rs @@ -0,0 +1,91 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use bytes::{BufMut, Bytes, BytesMut}; +use futures::Stream; +use log::*; + +pub struct ProtobufStream { + bytes_stream: Pin> + Send>>, + buf: BytesMut, + max_message_size: usize, + _marker: std::marker::PhantomData, +} + +impl ProtobufStream { + pub fn new(bytes_stream: impl Stream> + Send + 'static) -> Self { + Self::with_max_message_size(bytes_stream, 16 * 1024 * 1024) // Default to 16 MiB max message size + } + + pub fn with_max_message_size( + bytes_stream: impl Stream> + Send + 'static, + max_message_size: usize, + ) -> Self { + Self { + bytes_stream: Box::pin(bytes_stream), + buf: BytesMut::with_capacity(max_message_size), + max_message_size, + _marker: std::marker::PhantomData, + } + } +} + +impl Stream for ProtobufStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + match this.bytes_stream.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(bytes))) => { + this.buf.put(bytes); + + // Decode the length delimiter without advancing the buffer + // so that we can check if we have enough bytes or need to buffer more + let tmp_slice = &this.buf[..]; + // A length-delimited varint is complete once a byte with MSB 0 is seen (max 10 bytes for u64). + if tmp_slice.len() < 10 && tmp_slice.iter().take(10).all(|byte| byte & 0x80 != 0) { + // Need more bytes to finish reading the delimiter. + continue; + } + let len = prost::decode_length_delimiter(tmp_slice) + .map_err(|e| prost::DecodeError::new(format!("Failed to decode length delimiter: {}", e)))?; + if len > this.max_message_size { + return Poll::Ready(Some(Err(prost::DecodeError::new(format!( + "Message length {} exceeds maximum allowed size of {} bytes", + len, this.max_message_size + ))))); + } + + let len_delim_len = prost::length_delimiter_len(len); + if this.buf.len() < len + len_delim_len { + // Continue buffering + trace!( + "Buffering: have {} bytes, need {} bytes (including {} bytes for length delimiter)", + this.buf.len(), + len + len_delim_len, + len_delim_len + ); + continue; + } + + let msg = TMsg::decode_length_delimited(&mut this.buf)?; + break Poll::Ready(Some(Ok(msg))); + }, + Poll::Ready(Some(Err(e))) => { + break Poll::Ready(Some(Err(prost::DecodeError::new(format!( + "Error receiving bytes from stream: {}", + e + ))))); + }, + Poll::Ready(None) => break Poll::Ready(None), + Poll::Pending => break Poll::Pending, + } + } + } +} diff --git a/clients/tari_indexer_client/src/rest_api/mod.rs b/clients/tari_indexer_client/src/rest_api/mod.rs new file mode 100644 index 0000000000..480e616749 --- /dev/null +++ b/clients/tari_indexer_client/src/rest_api/mod.rs @@ -0,0 +1,4 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +pub mod types; diff --git a/clients/tari_indexer_client/src/rest_api/types.rs b/clients/tari_indexer_client/src/rest_api/types.rs new file mode 100644 index 0000000000..6b3e0240e9 --- /dev/null +++ b/clients/tari_indexer_client/src/rest_api/types.rs @@ -0,0 +1,22 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use multiaddr::Multiaddr; +use serde::{Deserialize, Serialize}; +use tari_template_lib_types::crypto::RistrettoPublicKeyBytes; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr( + feature = "ts", + derive(ts_rs::TS), + ts(export, export_to = "tari-indexer-client/", rename = "IndexerGetIdentityResponse") +)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +pub struct GetIdentityResponse { + pub peer_id: String, + #[cfg_attr(feature = "utoipa", schema(value_type = String))] + pub public_key: RistrettoPublicKeyBytes, + #[cfg_attr(feature = "utoipa", schema(value_type = Vec))] + #[cfg_attr(feature = "ts", ts(type = "string[]"))] + pub public_addresses: Vec, +} diff --git a/clients/tari_indexer_client/src/rest_api_client.rs b/clients/tari_indexer_client/src/rest_api_client.rs new file mode 100644 index 0000000000..78dfb30684 --- /dev/null +++ b/clients/tari_indexer_client/src/rest_api_client.rs @@ -0,0 +1,241 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use reqwest::{header, header::HeaderMap, IntoUrl, Url}; +use serde::{de::DeserializeOwned, Serialize}; +use tari_engine_types::substate::SubstateId; + +use crate::{ + error::IndexerRestClientError, + protobuf, + protobuf_stream::ProtobufStream, + types::{ + AddPeerRequest, + AddPeerResponse, + GetConnectionsResponse, + GetEpochManagerStatsResponse, + GetNetworkSyncStateResponse, + GetNonFungiblesRequest, + GetNonFungiblesResponse, + GetSubstateRequest, + GetSubstateResponse, + GetSubstatesRequest, + GetSubstatesResponse, + GetTemplateDefinitionRequest, + GetTemplateDefinitionResponse, + GetTransactionResultRequest, + GetTransactionResultResponse, + GetUnspentUtxosRequest, + GetUnspentUtxosResponse, + GetUtxoUpdatesRequest, + IndexerReadyResponse, + ListRecentTransactionsRequest, + ListRecentTransactionsResponse, + ListSubstatesRequest, + ListSubstatesResponse, + SubmitTransactionRequest, + SubmitTransactionResponse, + }, +}; + +#[derive(Debug, Clone)] +pub struct IndexerRestApiClient { + client: reqwest::Client, + endpoint: Url, +} + +impl IndexerRestApiClient { + pub fn connect(endpoint: T) -> Result { + let client = reqwest::Client::builder() + .default_headers({ + let mut headers = HeaderMap::with_capacity(1); + headers.insert(header::CONTENT_TYPE, "application/json".parse().unwrap()); + headers + }) + .build()?; + + Ok(Self { + client, + endpoint: endpoint.into_url()?, + }) + } + + pub async fn get_connections(&mut self) -> Result { + self.send_get("network/connections", ()).await + } + + pub async fn add_peer(&mut self, request: AddPeerRequest) -> Result { + self.send_post("network/connections", request).await + } + + pub async fn get_substate( + &mut self, + id: &SubstateId, + version: Option, + local_search_only: bool, + ) -> Result { + self.send_get(format!("substates/{id}"), GetSubstateRequest { + version, + local_search_only, + }) + .await + } + + pub async fn get_non_fungibles( + &mut self, + req: GetNonFungiblesRequest, + ) -> Result { + self.send_get("non-fungibles", req).await + } + + pub async fn fetch_substates( + &mut self, + req: GetSubstatesRequest, + ) -> Result { + self.send_post("substates/fetch", req).await + } + + pub async fn list_substates( + &mut self, + req: ListSubstatesRequest, + ) -> Result { + self.send_get("list_substates", req).await + } + + pub async fn submit_transaction( + &mut self, + req: SubmitTransactionRequest, + ) -> Result { + self.send_post("transactions", req).await + } + + pub async fn get_transaction_result( + &mut self, + req: GetTransactionResultRequest, + ) -> Result { + self.send_get(format!("transactions/{}/result", req.transaction_id), ()) + .await + } + + pub async fn list_recent_transactions( + &mut self, + req: ListRecentTransactionsRequest, + ) -> Result { + self.send_get("transactions/recent", req).await + } + + pub async fn get_template_definition( + &mut self, + req: GetTemplateDefinitionRequest, + ) -> Result { + self.send_get(format!("templates/{}", req.template_address), ()).await + } + + // pub async fn get_non_fungibles( + // &mut self, + // req: GetNonFungiblesRequest, + // ) -> Result { + // self.send_get("get_non_fungibles", req).await + // } + + pub async fn get_epoch_manager_stats(&mut self) -> Result { + self.send_get("epoch-manager/stats", ()).await + } + + pub async fn stream_utxo_updates_protobuf( + &mut self, + req: GetUtxoUpdatesRequest, + ) -> Result, IndexerRestClientError> { + const PATH: &str = "utxos/stream"; + let url = format!("{}{}", self.endpoint, PATH); + + let resp = self + .client + .post(&url) + .header(header::ACCEPT, "application/x-protobuf") + .json(&req) + .send() + .await?; + + let stream = resp.bytes_stream(); + let stream = ProtobufStream::::new(stream); + Ok(stream) + } + + pub async fn get_unspent_utxos( + &mut self, + req: GetUnspentUtxosRequest, + ) -> Result { + self.send_post("utxos/fetch", req).await + } + + pub async fn get_network_sync_state(&mut self) -> Result { + self.send_get("network/stats", ()).await + } + + pub async fn wait_until_ready(&mut self) -> Result { + self.send_get("wait-until-ready", ()).await + } + + async fn send_get, T: Serialize, R: DeserializeOwned>( + &mut self, + path: P, + params: T, + ) -> Result { + let path = path.into(); + + // encode query params + let query = serde_urlencoded::to_string(¶ms).map_err(|e| IndexerRestClientError::SerializeRequest { + path: path.clone(), + source: e.into(), + })?; + + let mut url = format!("{}{}", self.endpoint, path); + if !query.is_empty() { + url.push('?'); + url.push_str(&query); + } + + let resp = self.client.get(url).send().await?; + handle_json_response(resp, path).await + } + + async fn send_post, T: Serialize, R: DeserializeOwned>( + &mut self, + path: P, + request: T, + ) -> Result { + let path = path.into(); + + let resp = self + .client + .post(format!("{}{}", self.endpoint, path)) + .json(&request) + .send() + .await?; + + handle_json_response(resp, path).await + } +} + +async fn handle_json_response( + resp: reqwest::Response, + path: String, +) -> Result { + if let Some(err) = resp.error_for_status_ref().err() { + if let Ok(err_resp) = resp.json::().await { + return Err(IndexerRestClientError::ErrorResponse { + source: err, + details: err_resp.get("error").and_then(|v| v.as_str()).map(|s| s.to_string()), + }); + } + return Err(IndexerRestClientError::ErrorResponse { + source: err, + details: None, + }); + } + match resp.json().await { + Ok(r) => Ok(r), + Err(e) => Err(IndexerRestClientError::DeserializeResponse { path, source: e.into() }), + } +} diff --git a/clients/tari_indexer_client/src/types.rs b/clients/tari_indexer_client/src/types.rs index 8c704de07e..2945a69522 100644 --- a/clients/tari_indexer_client/src/types.rs +++ b/clients/tari_indexer_client/src/types.rs @@ -6,7 +6,6 @@ use std::{collections::HashMap, time::Duration}; use bounded_vec::BoundedVec; use multiaddr::Multiaddr; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, Seq}; use tari_common_types::types::FixedHash; use tari_consensus_types::Decision; use tari_engine_types::{ @@ -65,8 +64,6 @@ pub struct ListSubstateItem { ts(export, export_to = "tari-indexer-client/", rename = "IndexerGetSubstateRequest") )] pub struct GetSubstateRequest { - #[cfg_attr(feature = "ts", ts(type = "string"))] - pub address: SubstateId, pub version: Option, #[serde(default)] pub local_search_only: bool, @@ -79,7 +76,6 @@ pub struct GetSubstateRequest { ts(export, export_to = "tari-indexer-client/", rename = "IndexerGetSubstateResponse") )] pub struct GetSubstateResponse { - pub address: SubstateId, pub version: u32, pub substate: SubstateValue, } @@ -108,7 +104,6 @@ pub struct InspectSubstateRequest { #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS), ts(export, export_to = "tari-indexer-client/"))] pub struct InspectSubstateResponse { - pub address: SubstateId, pub version: u32, pub substate: SubstateValue, } @@ -146,7 +141,7 @@ pub struct SubmitTransactionResponse { #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS), ts(export, export_to = "tari-indexer-client/"))] pub struct ListTemplatesRequest { - pub limit: u32, + pub limit: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -161,7 +156,8 @@ pub struct TemplateMetadata { pub name: String, pub address: TemplateAddress, /// SHA hash of binary - pub binary_sha: String, + #[cfg_attr(feature = "ts", ts(type = "number[]"))] + pub binary_sha: FixedHash, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -312,11 +308,11 @@ pub struct GetEpochManagerStatsResponse { pub current_epoch: Epoch, #[cfg_attr(feature = "ts", ts(type = "number"))] pub current_block_height: u64, - #[cfg_attr(feature = "ts", ts(type = "string"))] + #[cfg_attr(feature = "ts", ts(type = "number[]"))] pub current_block_hash: FixedHash, } -#[derive(Serialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] #[cfg_attr( feature = "ts", derive(ts_rs::TS), @@ -335,7 +331,7 @@ pub struct Connection { pub user_agent: Option, } -#[derive(Serialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] #[cfg_attr( feature = "ts", derive(ts_rs::TS), @@ -346,7 +342,7 @@ pub enum ConnectionDirection { Outbound, } -#[derive(Serialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] #[cfg_attr( feature = "ts", derive(ts_rs::TS), @@ -373,14 +369,13 @@ pub struct GetTemplateDefinitionResponse { #[cfg_attr(feature = "ts", derive(ts_rs::TS), ts(export, export_to = "tari-indexer-client/"))] pub struct IndexerReadyResponse {} -#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS), ts(export, export_to = "tari-indexer-client/"))] pub struct GetUtxoUpdatesRequest { - #[cfg_attr(feature = "ts", ts(as = "Vec<(Shard, StateVersion)>"))] - #[serde_as(as = "Seq<(_, _)>")] - pub shard_state_versions: HashMap, + pub shard_state_versions: Vec<(Shard, StateVersion)>, pub resource_address: ResourceAddress, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub unspent_only: bool, pub per_shard_limit: u32, } @@ -404,15 +399,7 @@ pub struct GetUnspentUtxosResponse { } #[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr( - feature = "ts", - derive(ts_rs::TS), - ts( - export, - export_to = "tari-indexer-client/", - rename = "IndexerGetEpochManagerStatsResponse" - ) -)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS), ts(export, export_to = "tari-indexer-client/"))] pub struct GetNetworkSyncStateResponse { pub network_desc: NetworkDescription, pub sync_progress: Option, diff --git a/clients/tari_indexer_client/tests/streaming.rs b/clients/tari_indexer_client/tests/streaming.rs new file mode 100644 index 0000000000..e08b4728a4 --- /dev/null +++ b/clients/tari_indexer_client/tests/streaming.rs @@ -0,0 +1,32 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use futures::TryStreamExt; +use tari_indexer_client::{rest_api_client::IndexerRestApiClient, types::GetUtxoUpdatesRequest}; +use tari_ootle_common_types::{NumPreshards, StateVersion}; + +#[tokio::test] +#[ignore = "Requires a running indexer listening on a specific port"] +async fn dev_test() { + let mut client = IndexerRestApiClient::connect("http://localhost:12017").unwrap(); + let mut stream = client + .stream_utxo_updates_protobuf(GetUtxoUpdatesRequest { + shard_state_versions: NumPreshards::current() + .all_shards_iter() + .map(|shard| (shard, StateVersion::zero())) + .collect(), + resource_address: "resource_0101010101010101010101010101010101010101010101010101010101010101" + .parse() + .unwrap(), + unspent_only: false, + per_shard_limit: 1000, + }) + .await + .unwrap(); + + let mut count = 0usize; + while let Some(msg) = stream.try_next().await.unwrap() { + count += 1; + eprintln!("{count} {:?} {:?} {:?}", msg.sos, msg.update, msg.eos); + } +} diff --git a/crates/common_types/src/array_utils.rs b/crates/common_types/src/array_utils.rs new file mode 100644 index 0000000000..eb44d20d8b --- /dev/null +++ b/crates/common_types/src/array_utils.rs @@ -0,0 +1,12 @@ +// Copyright 2025 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +pub fn copy_fixed_checked(bytes: &[u8]) -> Option +where [u8; SZ]: Into { + if bytes.len() != SZ { + return None; + } + let mut array = [0u8; SZ]; + array.copy_from_slice(&bytes[..SZ]); + Some(array.into()) +} diff --git a/crates/common_types/src/lib.rs b/crates/common_types/src/lib.rs index 817985a53e..5828ff3572 100644 --- a/crates/common_types/src/lib.rs +++ b/crates/common_types/src/lib.rs @@ -1,6 +1,7 @@ // Copyright 2022 The Tari Project // SPDX-License-Identifier: BSD-3-Clause +pub mod array_utils; pub mod base_layer_hashing; pub mod borsh; mod bytes; diff --git a/crates/state_store_rocksdb/src/utils.rs b/crates/state_store_rocksdb/src/utils.rs index 8a9c9f6529..a8a5f9b4ad 100644 --- a/crates/state_store_rocksdb/src/utils.rs +++ b/crates/state_store_rocksdb/src/utils.rs @@ -22,7 +22,7 @@ use std::io; -use tari_ootle_storage::time::{PrimitiveDateTime, UtcDateTime}; +use tari_ootle_storage::time::{OffsetDateTime, PrimitiveDateTime}; pub(crate) fn read_to_fixed(reader: &mut R) -> Option where @@ -41,6 +41,6 @@ pub(crate) fn read_n_bytes(reader: &mut R, n: usize) -> Option PrimitiveDateTime { - let now = UtcDateTime::now(); + let now = OffsetDateTime::now_utc(); PrimitiveDateTime::new(now.date(), now.time()) } diff --git a/crates/storage/src/global/template_db.rs b/crates/storage/src/global/template_db.rs index 9c0a15408c..6035659adb 100644 --- a/crates/storage/src/global/template_db.rs +++ b/crates/storage/src/global/template_db.rs @@ -28,7 +28,7 @@ use std::{ use tari_common_types::types::FixedHash; use tari_ootle_common_types::Epoch; use tari_template_lib::types::{crypto::RistrettoPublicKeyBytes, TemplateAddress}; -use time::{PrimitiveDateTime, UtcDateTime}; +use time::{OffsetDateTime, PrimitiveDateTime}; use crate::global::GlobalDbAdapter; @@ -118,7 +118,7 @@ impl DbTemplate { } fn now() -> PrimitiveDateTime { - let now = UtcDateTime::now(); + let now = OffsetDateTime::now_utc(); PrimitiveDateTime::new(now.date(), now.time()) } diff --git a/crates/template_lib/src/models/utxo.rs b/crates/template_lib/src/models/utxo.rs index e770d3fd61..c2b6e23de2 100644 --- a/crates/template_lib/src/models/utxo.rs +++ b/crates/template_lib/src/models/utxo.rs @@ -95,6 +95,10 @@ impl UtxoId { PedersenCommitmentBytes::from_array(self.0) } + pub fn as_bytes(&self) -> &[u8; Self::LENGTH] { + &self.0 + } + pub fn to_commitment_hex_string(&self) -> String { // to_string happens to return the hex encoding of the commitment bytes. If that changes, so will this. self.to_string() diff --git a/crates/template_lib_types/src/crypto/utxo_tag.rs b/crates/template_lib_types/src/crypto/utxo_tag.rs index 79f7a84a5f..6cef4da758 100644 --- a/crates/template_lib_types/src/crypto/utxo_tag.rs +++ b/crates/template_lib_types/src/crypto/utxo_tag.rs @@ -22,6 +22,12 @@ impl UtxoTag { } } +impl From for UtxoTag { + fn from(value: u32) -> Self { + Self(value) + } +} + impl fmt::Display for UtxoTag { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "UtxoTag({})", self.0) diff --git a/crates/template_manager/src/implementation/manager.rs b/crates/template_manager/src/implementation/manager.rs index c46acc8dfc..d46976b1c4 100644 --- a/crates/template_manager/src/implementation/manager.rs +++ b/crates/template_manager/src/implementation/manager.rs @@ -43,7 +43,7 @@ use tari_ootle_common_types::{ use tari_ootle_p2p::proto::rpc::TemplateType; use tari_ootle_storage::{ global::{DbTemplate, DbTemplateType, DbTemplateUpdate, GlobalDb, TemplateStatus}, - time::{PrimitiveDateTime, UtcDateTime}, + time::{OffsetDateTime, PrimitiveDateTime}, }; use tari_ootle_storage_sqlite::global::SqliteGlobalDbAdapter; use tari_template_builtin::{ @@ -439,6 +439,6 @@ impl Clone for TemplateManager { } fn now() -> PrimitiveDateTime { - let now = UtcDateTime::now(); + let now = OffsetDateTime::now_utc(); PrimitiveDateTime::new(now.date(), now.time()) } diff --git a/crates/transaction/src/special_json_arg_syntax.rs b/crates/transaction/src/special_json_arg_syntax.rs index a46e8f526d..d5a57e879c 100644 --- a/crates/transaction/src/special_json_arg_syntax.rs +++ b/crates/transaction/src/special_json_arg_syntax.rs @@ -61,7 +61,7 @@ fn is_arg_json(arg: &json::Value) -> bool { }; if let Some(lit) = obj.get("Literal") { - // Support for {"Literal" "deadbeaf"} - common case for wallet -> indexer JSON rpc + // Support for {"Literal" "deadbeaf"} - common case for wallet -> indexer api if let Some(s) = lit.as_str() { return s.chars().all(|c| c.is_ascii_hexdigit()); } diff --git a/crates/wallet/sdk/Cargo.toml b/crates/wallet/sdk/Cargo.toml index 7a669f40eb..ecae5c04ca 100644 --- a/crates/wallet/sdk/Cargo.toml +++ b/crates/wallet/sdk/Cargo.toml @@ -27,6 +27,7 @@ blake2 = { workspace = true } digest = { workspace = true } log = { workspace = true } serde = { workspace = true, default-features = true } +futures = { workspace = true } time = { workspace = true, features = ["serde", "serde-human-readable"] } thiserror = { workspace = true } ts-rs = { workspace = true, optional = true } diff --git a/crates/wallet/sdk/src/apis/stealth_transfer.rs b/crates/wallet/sdk/src/apis/stealth_transfer.rs index 4dcf5426e6..53afe881f1 100644 --- a/crates/wallet/sdk/src/apis/stealth_transfer.rs +++ b/crates/wallet/sdk/src/apis/stealth_transfer.rs @@ -611,7 +611,7 @@ where self.outputs_api.locks_set_transaction_id(lock_id, tx_id)?; Ok(TransferOutput { transaction, - transaction_lock_id: Some(lock_id), + transaction_lock_id: lock_id, }) }, Err(err) => { @@ -852,7 +852,7 @@ where pub struct TransferOutput { pub transaction: Transaction, - pub transaction_lock_id: Option, + pub transaction_lock_id: WalletLockId, } #[derive(Debug)] diff --git a/crates/wallet/sdk/src/models/utxo_update.rs b/crates/wallet/sdk/src/models/utxo_update.rs index ba7d91c879..8793d06aec 100644 --- a/crates/wallet/sdk/src/models/utxo_update.rs +++ b/crates/wallet/sdk/src/models/utxo_update.rs @@ -10,6 +10,25 @@ use tari_template_lib::{ types::crypto::{RistrettoPublicKeyBytes, UtxoTag}, }; +#[derive(Debug, Clone)] +pub struct UtxoUpdatePayload { + pub sos: Option, + pub update: Option, + pub eos: Option, +} + +#[derive(Debug, Clone)] +pub struct StartOfShard { + pub shard: Shard, + pub max_state_version: StateVersion, + pub has_more: bool, +} + +#[derive(Debug, Clone)] +pub struct EndOfShard { + pub max_state_version: StateVersion, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS), ts(export))] pub struct UtxoUpdateSet { diff --git a/crates/wallet/sdk/src/network.rs b/crates/wallet/sdk/src/network.rs index 63d02e838c..81276d79ae 100644 --- a/crates/wallet/sdk/src/network.rs +++ b/crates/wallet/sdk/src/network.rs @@ -1,8 +1,9 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::{collections::HashMap, convert::Infallible, future::Future, time::Duration}; +use std::{collections::HashMap, convert::Infallible, future::Future, pin::Pin, time::Duration}; +use futures::Stream; use serde::{Deserialize, Serialize}; use tari_consensus_types::Decision; use tari_engine_types::{ @@ -20,7 +21,10 @@ use tari_template_lib::{ use tari_transaction::{Transaction, TransactionId}; use time::PrimitiveDateTime; -use crate::models::UtxoUpdateSet; +use crate::models::UtxoUpdatePayload; + +/// A pinned, boxed stream of UTXO updates +pub type UtxoUpdateStream = Pin> + Send + 'static>>; pub trait WalletNetworkInterface { type Error: std::error::Error + Send + Sync + 'static; @@ -57,11 +61,12 @@ pub trait WalletNetworkInterface { template_address: TemplateAddress, ) -> impl Future> + Send; - fn query_stealth_utxo_updates( + fn stream_stealth_utxo_updates( &self, resource_address: ResourceAddress, - shard_state_versions: HashMap, - ) -> impl Future> + Send; + shard_state_versions: Vec<(Shard, StateVersion)>, + unspent_only: bool, + ) -> impl Future, Self::Error>> + Send; fn get_unspent_utxos( &self, @@ -101,7 +106,6 @@ pub enum WalletQueryErrorStatus { #[derive(Debug, Clone, Deserialize, Serialize)] pub struct SubstateQueryResult { - pub address: SubstateId, pub version: u32, pub substate: SubstateValue, } diff --git a/crates/wallet/sdk/tests/support/harness.rs b/crates/wallet/sdk/tests/support/harness.rs index 32bf1af906..9ef00b2921 100644 --- a/crates/wallet/sdk/tests/support/harness.rs +++ b/crates/wallet/sdk/tests/support/harness.rs @@ -14,7 +14,7 @@ use tari_ootle_common_types::{optional::Optional, shard::Shard, Network, StateVe use tari_ootle_wallet_sdk::{ cipher_seed::CipherSeedRestore, models::{ConfidentialOutputModel, KeyId, OutputStatus, UtxoUpdateSet, WalletLockId}, - network::{SubstateQueryResult, TransactionQueryResult, WalletNetworkInterface}, + network::{SubstateQueryResult, TransactionQueryResult, UtxoUpdateStream, WalletNetworkInterface}, storage::TagAndPublicNoncePair, WalletSdk, WalletSdkConfig, @@ -181,11 +181,12 @@ impl WalletNetworkInterface for PanicNetworkInterface { panic!("PanicNetworkInterface called") } - async fn query_stealth_utxo_updates( + async fn stream_stealth_utxo_updates( &self, _resource_address: ResourceAddress, - _shard_state_versions: HashMap, - ) -> Result { + _shard_state_versions: Vec<(Shard, StateVersion)>, + _unspent_only: bool, + ) -> Result, Self::Error> { panic!("PanicNetworkInterface called") } diff --git a/crates/wallet/sdk_services/Cargo.toml b/crates/wallet/sdk_services/Cargo.toml index afc9ce007e..37774f993d 100644 --- a/crates/wallet/sdk_services/Cargo.toml +++ b/crates/wallet/sdk_services/Cargo.toml @@ -23,6 +23,7 @@ tari_shutdown = { workspace = true } anyhow = { workspace = true } reqwest = { workspace = true, optional = true } +futures = { workspace = true } futures-bounded = { workspace = true } thiserror = { workspace = true } url = { workspace = true } @@ -31,4 +32,4 @@ tokio = { workspace = true, features = ["sync", "macros", "time"] } [features] default = [] -indexer_jrpc = ["tari_indexer_client", "reqwest"] \ No newline at end of file +indexer_client = ["tari_indexer_client", "reqwest"] \ No newline at end of file diff --git a/crates/wallet/sdk_services/src/indexer_jrpc.rs b/crates/wallet/sdk_services/src/indexer_rest_api.rs similarity index 53% rename from crates/wallet/sdk_services/src/indexer_jrpc.rs rename to crates/wallet/sdk_services/src/indexer_rest_api.rs index aaeb7f47e5..cfc95aadca 100644 --- a/crates/wallet/sdk_services/src/indexer_jrpc.rs +++ b/crates/wallet/sdk_services/src/indexer_rest_api.rs @@ -6,16 +6,18 @@ use std::{ sync::{Arc, Mutex}, }; +use anyhow::anyhow; +use futures::{StreamExt, TryStreamExt}; use reqwest::{IntoUrl, Url}; use tari_engine_types::{ substate::{Substate, SubstateId}, Utxo, }; use tari_indexer_client::{ - error::IndexerClientError, - json_rpc_client::IndexerJsonRpcClient, + error::IndexerRestClientError, + protobuf, + rest_api_client::IndexerRestApiClient, types::{ - GetSubstateRequest, GetSubstatesRequest, GetTransactionResultRequest, GetUnspentUtxosRequest, @@ -24,14 +26,15 @@ use tari_indexer_client::{ SubmitTransactionRequest, }, }; -use tari_ootle_common_types::{optional::IsNotFoundError, shard::Shard, StateVersion}; +use tari_ootle_common_types::{array_utils::copy_fixed_checked, optional::IsNotFoundError, shard::Shard, StateVersion}; use tari_ootle_wallet_sdk::{ - models::UtxoUpdateSet, + models::{EndOfShard, StartOfShard, UtxoBurnt, UtxoSpent, UtxoUnspent, UtxoUpdatePayload, WalletUtxoUpdate}, network::{ StatusResponseError, SubstateQueryResult, TransactionFinalizedResult, TransactionQueryResult, + UtxoUpdateStream, WalletNetworkInterface, WalletQueryErrorStatus, }, @@ -47,38 +50,34 @@ use url::ParseError; const INVALID_REQUEST_CODE: i64 = 400; #[derive(Debug, Clone)] -pub struct IndexerJsonRpcNetworkInterface { - indexer_jrpc_address: Arc>, +pub struct IndexerRestApiNetworkInterface { + url: Arc>, } -impl IndexerJsonRpcNetworkInterface { - pub fn new(indexer_jrpc_address: T) -> Self { +impl IndexerRestApiNetworkInterface { + pub fn new(url: T) -> Self { Self { - indexer_jrpc_address: Arc::new(Mutex::new( - indexer_jrpc_address - .into_url() - .expect("Malformed indexer JSON-RPC address"), - )), + url: Arc::new(Mutex::new(url.into_url().expect("Malformed indexer JSON-RPC address"))), } } - fn get_client(&self) -> Result { - let client = IndexerJsonRpcClient::connect((*self.indexer_jrpc_address.lock().unwrap()).clone())?; + fn get_client(&self) -> Result { + let client = IndexerRestApiClient::connect((*self.url.lock().unwrap()).clone())?; Ok(client) } - pub fn set_endpoint(&self, endpoint: &str) -> Result<(), IndexerJrpcError> { - *self.indexer_jrpc_address.lock().unwrap() = Url::parse(endpoint)?; + pub fn set_endpoint(&self, endpoint: &str) -> Result<(), IndexerRestApiNetworkInterfaceError> { + *self.url.lock().unwrap() = Url::parse(endpoint)?; Ok(()) } pub fn get_endpoint(&self) -> Url { - (*self.indexer_jrpc_address.lock().unwrap()).clone() + (*self.url.lock().unwrap()).clone() } } -impl WalletNetworkInterface for IndexerJsonRpcNetworkInterface { - type Error = IndexerJrpcError; +impl WalletNetworkInterface for IndexerRestApiNetworkInterface { + type Error = IndexerRestApiNetworkInterfaceError; async fn query_substate( &self, @@ -87,15 +86,8 @@ impl WalletNetworkInterface for IndexerJsonRpcNetworkInterface { local_search_only: bool, ) -> Result { let mut client = self.get_client()?; - let result = client - .get_substate(GetSubstateRequest { - address: substate_id.clone(), - version, - local_search_only, - }) - .await?; + let result = client.get_substate(substate_id, version, local_search_only).await?; Ok(SubstateQueryResult { - address: result.address, version: result.version, substate: result.substate, }) @@ -104,12 +96,16 @@ impl WalletNetworkInterface for IndexerJsonRpcNetworkInterface { async fn get_substates(&self, substate_ids: Vec) -> Result, Self::Error> { let mut client = self.get_client()?; let resp = client - .get_substates(GetSubstatesRequest { + .fetch_substates(GetSubstatesRequest { requests: substate_ids.try_into().map_err(|_| { - IndexerJrpcError::IndexerClientError(IndexerClientError::RequestFailedWithStatus { - code: INVALID_REQUEST_CODE, - message: "Too many substate IDs requested".to_string(), - }) + // We can't send the request, but return an error that would be the same/similar to what the indexer + // would return + IndexerRestApiNetworkInterfaceError::IndexerClientError( + IndexerRestClientError::RequestFailedWithStatus { + code: INVALID_REQUEST_CODE, + message: "Too many substate IDs requested".to_string(), + }, + ) })?, }) .await?; @@ -173,20 +169,75 @@ impl WalletNetworkInterface for IndexerJsonRpcNetworkInterface { Ok(resp.definition) } - async fn query_stealth_utxo_updates( + async fn stream_stealth_utxo_updates( &self, resource_address: ResourceAddress, - shard_state_versions: HashMap, - ) -> Result { + shard_state_versions: Vec<(Shard, StateVersion)>, + unspent_only: bool, + ) -> Result, Self::Error> { let mut client = self.get_client()?; - let resp = client - .get_utxo_updates(GetUtxoUpdatesRequest { + let stream = client + .stream_utxo_updates_protobuf(GetUtxoUpdatesRequest { shard_state_versions, resource_address, - per_shard_limit: 100, + unspent_only, + per_shard_limit: 1000, }) .await?; - Ok(resp.updates) + let stream = stream + .map_err(|e| IndexerRestApiNetworkInterfaceError::StreamDecodeError(e.into())) + .and_then(|res| async move { + let sos = res.sos.map(|sos| StartOfShard { + shard: Shard::from(sos.shard), + max_state_version: StateVersion::from(sos.max_state_version), + has_more: sos.num_updates >= 1000, + }); + let update = res + .update + .map(|u| match u { + protobuf::WalletUtxoUpdate::Unspent(unspent) => { + let public_nonce = + RistrettoPublicKeyBytes::from_bytes(&unspent.public_nonce).map_err(|e| { + IndexerRestApiNetworkInterfaceError::StreamDecodeError(anyhow!( + "Failed to decode public nonce: {e}" + )) + })?; + Ok::<_, IndexerRestApiNetworkInterfaceError>(WalletUtxoUpdate::Unspent(UtxoUnspent { + tag: unspent.tag.into(), + public_nonce, + })) + }, + protobuf::WalletUtxoUpdate::Spent(spent) => { + let id_arr = copy_fixed_checked(&spent.id).ok_or_else(|| { + IndexerRestApiNetworkInterfaceError::StreamDecodeError(anyhow!( + "Failed to decode UTXO ID, incorrect length" + )) + })?; + Ok::<_, IndexerRestApiNetworkInterfaceError>(WalletUtxoUpdate::Spent(UtxoSpent { + id: UtxoId::from_array(id_arr), + version: spent.version, + })) + }, + protobuf::WalletUtxoUpdate::Burnt(burnt) => { + let id_arr = copy_fixed_checked(&burnt.id).ok_or_else(|| { + IndexerRestApiNetworkInterfaceError::StreamDecodeError(anyhow!( + "Failed to decode UTXO ID, incorrect length" + )) + })?; + Ok::<_, IndexerRestApiNetworkInterfaceError>(WalletUtxoUpdate::Burnt(UtxoBurnt { + id: UtxoId::from_array(id_arr), + version: burnt.version, + })) + }, + }) + .transpose()?; + let eos = res.eos.map(|eos| EndOfShard { + max_state_version: eos.max_state_version.into(), + }); + + Ok(UtxoUpdatePayload { sos, update, eos }) + }); + Ok(stream.boxed()) } async fn get_unspent_utxos( @@ -212,38 +263,42 @@ impl WalletNetworkInterface for IndexerJsonRpcNetworkInterface { } #[derive(Debug, thiserror::Error)] -pub enum IndexerJrpcError { +pub enum IndexerRestApiNetworkInterfaceError { #[error("Indexer client error: {0}")] - IndexerClientError(#[from] IndexerClientError), + IndexerClientError(#[from] IndexerRestClientError), #[error("Indexer parse error : {0}")] IndexerParseError(#[from] ParseError), + #[error("Stream decode error: {0}")] + StreamDecodeError(anyhow::Error), } -impl IsNotFoundError for IndexerJrpcError { +impl IsNotFoundError for IndexerRestApiNetworkInterfaceError { fn is_not_found_error(&self) -> bool { match self { - IndexerJrpcError::IndexerClientError(err) => err.is_not_found_error(), + IndexerRestApiNetworkInterfaceError::IndexerClientError(err) => err.is_not_found_error(), _ => false, } } } -impl StatusResponseError for IndexerJrpcError { +impl StatusResponseError for IndexerRestApiNetworkInterfaceError { fn get_status(&self) -> WalletQueryErrorStatus { match self { - IndexerJrpcError::IndexerClientError(err) => { + IndexerRestApiNetworkInterfaceError::IndexerClientError(err) => { if err.is_not_found_error() { return WalletQueryErrorStatus::NotFound { message: "The requested resource was not found".to_string(), }; } match err { - IndexerClientError::RequestFailedWithStatus { code, message } if *code == INVALID_REQUEST_CODE => { + IndexerRestClientError::RequestFailedWithStatus { code, message } + if *code == INVALID_REQUEST_CODE => + { WalletQueryErrorStatus::TransactionRejected { message: message.clone(), } }, - IndexerClientError::RequestFailedWithStatus { code, message } => { + IndexerRestClientError::RequestFailedWithStatus { code, message } => { WalletQueryErrorStatus::InternalError { message: format!("Indexer request failed with status {code}: {message}"), } @@ -253,9 +308,12 @@ impl StatusResponseError for IndexerJrpcError { }, } }, - IndexerJrpcError::IndexerParseError(e) => WalletQueryErrorStatus::InternalError { + IndexerRestApiNetworkInterfaceError::IndexerParseError(e) => WalletQueryErrorStatus::InternalError { message: format!("Indexer parse error: {e}"), }, + IndexerRestApiNetworkInterfaceError::StreamDecodeError(e) => WalletQueryErrorStatus::InternalError { + message: format!("Indexer stream decode error: {e}"), + }, } } diff --git a/crates/wallet/sdk_services/src/lib.rs b/crates/wallet/sdk_services/src/lib.rs index 1460bf2af8..ec3011cc42 100644 --- a/crates/wallet/sdk_services/src/lib.rs +++ b/crates/wallet/sdk_services/src/lib.rs @@ -4,8 +4,8 @@ pub mod account_monitor; pub mod account_recovery; pub mod events; -#[cfg(feature = "indexer_jrpc")] -pub mod indexer_jrpc; +#[cfg(feature = "indexer_client")] +pub mod indexer_rest_api; pub mod notify; pub mod transaction_service; pub mod utxo_scanner; diff --git a/crates/wallet/sdk_services/src/utxo_scanner/scanner_round.rs b/crates/wallet/sdk_services/src/utxo_scanner/scanner_round.rs index 7d4f01f6bd..feb6b08bd1 100644 --- a/crates/wallet/sdk_services/src/utxo_scanner/scanner_round.rs +++ b/crates/wallet/sdk_services/src/utxo_scanner/scanner_round.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; +use futures::StreamExt; use log::{debug, info, trace, warn}; use tari_crypto::ristretto::RistrettoPublicKey; use tari_engine_types::ConvertFromByteType; @@ -14,8 +15,8 @@ use tari_ootle_common_types::{ StateVersion, }; use tari_ootle_wallet_sdk::{ - models::{AccountWithAddress, Key, UtxoSpent, UtxoUnspent, WalletUtxoUpdate}, - network::{StatusResponseError, WalletNetworkInterface}, + models::{AccountWithAddress, Key, StartOfShard, UtxoSpent, UtxoUnspent, WalletUtxoUpdate}, + network::{StatusResponseError, UtxoUpdateStream, WalletNetworkInterface}, storage::{WalletStorageError, WalletStore, WalletStoreReader, WalletStoreWriter}, WalletSdk, }; @@ -95,7 +96,7 @@ where .store() .with_read_tx(|tx| tx.shard_state_version_get(self.account.component_address(), self.resource_address))?; - // Populate any missing shards with zero + // Populate any missing shards with zero because we want to scan all shards for shard in NUM_PRESHARDS.all_shards_iter() { shard_state_versions.entry(shard).or_insert_with(StateVersion::zero); } @@ -108,44 +109,67 @@ where shard_state_versions.len(), ); - let response = self + let stream = self .sdk .get_network_interface() - .query_stealth_utxo_updates(*self.resource_address, shard_state_versions) + .stream_stealth_utxo_updates( + *self.resource_address, + // NOTE that this will request shards in a random order (HashMap). This is good to avoid always + // starting with the same shard. + shard_state_versions.into_iter().collect(), + false, + ) .await .map_err(|e| StealthScannerApiError::NetworkInterfaceError(e.into()))?; - // Keep going until there are no more - if response.shard_updates.is_empty() { - info!( - target: LOG_TARGET, - "🔍️ Scan complete for account {}: No more stealth outputs found", - self.account.component_address() - ); - // Update state versions to avoid rescanning from previous versions that didnt contain any changes - self.sdk.store().with_write_tx(|tx| { - tx.shard_state_version_set_many( - self.account.component_address(), - self.resource_address, - response.per_shard_high_watermark, - ) - })?; + let has_more = self.process_stream(stream).await?; + Ok(has_more) + } - return Ok(false); - } + async fn process_stream( + &mut self, + mut stream: UtxoUpdateStream, + ) -> Result { + let mut num_received = 0usize; + let mut sos: Option = None; + let mut has_more = false; + + while let Some(result) = stream.next().await { + let update = result.map_err(|e| StealthScannerApiError::NetworkInterfaceError(e.into()))?; + if let Some(recv_sos) = update.sos { + debug!(target: LOG_TARGET, "Received SOS for shard {}", recv_sos.shard); + // Commit the previous progress and start with the next shard + if let Some(sos) = sos.take() { + if num_received > 0 { + info!( + target: LOG_TARGET, + "🔍️ Scan complete for account {}: No more stealth outputs found in shard {} (max state version {})", + self.account.component_address(), + sos.shard, + sos.max_state_version + ); + } + self.stats.num_received += num_received; + num_received = 0; + self.commit_progress()?; + } - let num_received = response.shard_updates.len(); + self.shard_state_versions_to_set + .entry(recv_sos.shard) + .and_modify(|v| { + if recv_sos.max_state_version > *v { + *v = recv_sos.max_state_version + } + }) + .or_insert(recv_sos.max_state_version); - for (shard, update_set) in response.shard_updates { - self.shard_state_versions_to_set - .entry(shard) - .and_modify(|v| { - if update_set.max_state_version > *v { - *v = update_set.max_state_version - } - }) - .or_insert(update_set.max_state_version); - for update in update_set.updates { + has_more |= recv_sos.has_more; + + sos = Some(recv_sos); + } + + if let Some(update) = update.update { + num_received += 1; match update { WalletUtxoUpdate::Unspent(unspent) => { if self.check_if_tag_matches(&unspent)? { @@ -172,11 +196,31 @@ where } } - // Atomically persist all changes from this round - let num_recovered = self.utxos_to_recover.len(); + self.commit_progress()?; self.stats.num_received += num_received; - self.stats.num_potential_recoveries += num_recovered; + + if !has_more { + info!( + target: LOG_TARGET, + "🔍️ Scan complete for account {}: No more stealth outputs found", + self.account.component_address() + ); + } + + Ok(has_more) + } + + fn commit_progress(&mut self) -> Result { + if self.utxos_to_recover.is_empty() && + self.utxos_to_spend.is_empty() && + self.shard_state_versions_to_set.is_empty() + { + return Ok(false); + } + let mut num_spent = 0; + let num_recovered = self.utxos_to_recover.len(); + self.stats.num_potential_recoveries += num_recovered; self.sdk.store().with_write_tx(|tx| { // Mark UTXOs as spent (if they exist) for spent in self.utxos_to_spend.drain(..) { @@ -188,28 +232,23 @@ where } } - // Queue up tag matching UTXOs for processing - tx.utxo_process_queue_extend(self.resource_address, self.utxos_to_recover.drain(..))?; + if !self.utxos_to_recover.is_empty() { + // Queue up tag matching UTXOs for processing + tx.utxo_process_queue_extend(self.resource_address, self.utxos_to_recover.drain(..))?; + } // Update shard state versions - tx.shard_state_version_set_many( - self.account.component_address(), - self.resource_address, - self.shard_state_versions_to_set.drain(), - ) + if !self.shard_state_versions_to_set.is_empty() { + tx.shard_state_version_set_many( + self.account.component_address(), + self.resource_address, + self.shard_state_versions_to_set.drain(), + )?; + } + Ok::<_, StealthScannerApiError>(()) })?; - info!( - target: LOG_TARGET, - "Scan round complete for account {}: Validated the tag of {}/{} new stealth outputs, marked {} as spent", - self.account.component_address(), - num_recovered, - num_received, - num_spent - ); - self.stats.num_spent += num_spent; - Ok(true) } diff --git a/integration_tests/src/indexer.rs b/integration_tests/src/indexer.rs index 532e9a7280..0d2a094e5d 100644 --- a/integration_tests/src/indexer.rs +++ b/integration_tests/src/indexer.rs @@ -34,8 +34,8 @@ use tari_indexer::{ }; use tari_indexer_client::{ graphql_client::IndexerGraphQLClient, - json_rpc_client::IndexerJsonRpcClient, - types::{AddPeerRequest, GetNonFungiblesRequest, GetSubstateRequest, GetSubstateResponse, NonFungibleSubstate}, + rest_api_client::IndexerRestApiClient, + types::{AddPeerRequest, GetNonFungiblesRequest, GetSubstateResponse, NonFungibleSubstate}, }; use tari_ootle_app_utilities::{epoch_oracle_config::EpochOracleConfig, p2p_config::PeerSeedsConfig}; use tari_ootle_common_types::Network; @@ -54,7 +54,7 @@ use crate::{ pub struct IndexerProcess { pub name: String, pub port: u16, - pub json_rpc_port: u16, + pub api_port: u16, pub graphql_port: u16, pub base_node_grpc_port: u16, pub web_ui_port: u16, @@ -66,8 +66,8 @@ pub struct IndexerProcess { impl IndexerProcess { pub async fn add_peer(&self, public_key: RistrettoPublicKeyBytes, port: u16) { - let mut jrpc_client = self.get_jrpc_indexer_client(); - jrpc_client + let mut client = self.get_indexer_client(); + client .add_peer(AddPeerRequest { public_key, addresses: vec![multiaddr!(Ip4([127, 0, 0, 1]), Tcp(port))], @@ -79,16 +79,8 @@ impl IndexerProcess { pub async fn get_substate(&self, world: &TariWorld, output_ref: String, version: u32) -> GetSubstateResponse { let address = get_address_from_output(world, output_ref); - - let mut jrpc_client = self.get_jrpc_indexer_client(); - jrpc_client - .get_substate(GetSubstateRequest { - address: address.clone(), - version: Some(version), - local_search_only: true, - }) - .await - .unwrap() + let mut client = self.get_indexer_client(); + client.get_substate(address, Some(version), true).await.unwrap() } pub async fn get_non_fungibles( @@ -110,14 +102,14 @@ impl IndexerProcess { end_index, }; - let mut jrpc_client = self.get_jrpc_indexer_client(); - let resp = jrpc_client.get_non_fungibles(params).await.unwrap(); + let mut client = self.get_indexer_client(); + let resp = client.get_non_fungibles(params).await.unwrap(); resp.non_fungibles } - pub fn get_jrpc_indexer_client(&self) -> IndexerJsonRpcClient { - let endpoint: Url = Url::parse(&format!("http://localhost:{}", self.json_rpc_port)).unwrap(); - IndexerJsonRpcClient::connect(endpoint).unwrap() + pub fn get_indexer_client(&self) -> IndexerRestApiClient { + let endpoint: Url = Url::parse(&format!("http://localhost:{}", self.api_port)).unwrap(); + IndexerRestApiClient::connect(endpoint).unwrap() } pub async fn get_graphql_indexer_client(&self) -> IndexerGraphQLClient { @@ -128,7 +120,7 @@ impl IndexerProcess { pub async fn spawn_indexer(world: &mut TariWorld, indexer_name: String, base_node_name: String) { // each spawned indexer will use different ports - let (port, json_rpc_port) = get_os_assigned_ports(); + let (port, api_port) = get_os_assigned_ports(); let (graphql_port, web_ui_port) = get_os_assigned_ports(); let base_node_grpc_port = world.base_nodes.get(&base_node_name).unwrap().grpc_port; let name = indexer_name.clone(); @@ -168,7 +160,7 @@ pub async fn spawn_indexer(world: &mut TariWorld, indexer_name: String, base_nod config.indexer.p2p.listener_port = port; config.indexer.p2p.enable_mdns = false; - config.indexer.json_rpc_address = Some(format!("127.0.0.1:{}", json_rpc_port).parse().unwrap()); + config.indexer.api_listen_address = Some(format!("127.0.0.1:{}", api_port).parse().unwrap()); config.indexer.web_ui_address = Some(format!("127.0.0.1:{}", web_ui_port).parse().unwrap()); config.indexer.graphql_address = Some(format!("127.0.0.1:{}", graphql_port).parse().unwrap()); @@ -182,7 +174,7 @@ pub async fn spawn_indexer(world: &mut TariWorld, indexer_name: String, base_nod }); // Wait for node to start up - let handle = wait_listener_on_local_port(handle, json_rpc_port).await; + let handle = wait_listener_on_local_port(handle, api_port).await; // Check if the task errored/panicked let handle = check_join_handle(&name, handle).await; @@ -193,7 +185,7 @@ pub async fn spawn_indexer(world: &mut TariWorld, indexer_name: String, base_nod base_node_grpc_port, web_ui_port, handle, - json_rpc_port, + api_port, graphql_port, temp_dir_path: base_dir_path, shutdown, diff --git a/integration_tests/src/lib.rs b/integration_tests/src/lib.rs index 0496dbec02..ce1717c538 100644 --- a/integration_tests/src/lib.rs +++ b/integration_tests/src/lib.rs @@ -364,7 +364,7 @@ impl TariWorld { for (name, node) in &self.indexers { eprintln!( "Indexer \"{}\": json rpc port \"{}\", graphql port \"{}\", web ui port \"{}\", temp dir path \"{}\"", - name, node.json_rpc_port, node.graphql_port, node.web_ui_port, node.temp_dir_path + name, node.api_port, node.graphql_port, node.web_ui_port, node.temp_dir_path ); } @@ -384,8 +384,8 @@ impl TariWorld { // wallet daemons for (name, daemon) in &self.wallet_daemons { eprintln!( - "Wallet daemon \"{}\": json rpc port \"{}\", indexer jrpc port \"{}\", temp dir path \"{:?}\"", - name, daemon.json_rpc_port, daemon.indexer_jrpc_port, daemon.temp_path_dir + "Wallet daemon \"{}\": json rpc port \"{}\", indexer api port \"{}\", temp dir path \"{:?}\"", + name, daemon.json_rpc_port, daemon.indexer_api_port, daemon.temp_path_dir ); } diff --git a/integration_tests/src/wallet_daemon.rs b/integration_tests/src/wallet_daemon.rs index 4228b35d49..037900ddc9 100644 --- a/integration_tests/src/wallet_daemon.rs +++ b/integration_tests/src/wallet_daemon.rs @@ -62,7 +62,7 @@ use crate::{ pub struct TariWalletDaemonProcess { pub name: String, pub json_rpc_port: u16, - pub indexer_jrpc_port: u16, + pub indexer_api_port: u16, pub temp_path_dir: PathBuf, pub shutdown: Shutdown, } @@ -71,13 +71,13 @@ pub async fn spawn_wallet_daemon(world: &mut TariWorld, wallet_daemon_name: Stri let (signaling_server_port, json_rpc_port) = get_os_assigned_ports(); let base_dir = get_base_dir_for_scenario("wallet_daemon", world.get_current_scenario_name(), &wallet_daemon_name); - let indexer_jrpc_port = world.get_indexer(&indexer_name).json_rpc_port; + let indexer_api_port = world.get_indexer(&indexer_name).api_port; let shutdown = Shutdown::new(); let shutdown_signal = shutdown.to_signal(); let json_rpc_address = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), json_rpc_port); let signaling_server_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), signaling_server_port); - let indexer_url = format!("http://127.0.0.1:{}/json_rpc", indexer_jrpc_port); + let indexer_url = format!("http://127.0.0.1:{}", indexer_api_port); let mut config = ApplicationConfig { common: CommonConfig::default(), @@ -87,7 +87,7 @@ pub async fn spawn_wallet_daemon(world: &mut TariWorld, wallet_daemon_name: Stri config.common.base_path.clone_from(&base_dir); config.ootle_wallet_daemon.json_rpc_address = Some(json_rpc_address); config.ootle_wallet_daemon.signaling_server_address = Some(signaling_server_addr); - config.ootle_wallet_daemon.indexer_json_rpc_url = indexer_url.parse().unwrap(); + config.ootle_wallet_daemon.indexer_api_url = indexer_url.parse().unwrap(); config.ootle_wallet_daemon.network = Network::LocalNet; config.ootle_wallet_daemon.authentication = WalletDaemonAuth::None; config.ootle_wallet_daemon.override_keyring_password = Some("secret".into()); @@ -103,7 +103,7 @@ pub async fn spawn_wallet_daemon(world: &mut TariWorld, wallet_daemon_name: Stri let wallet_daemon_process = TariWalletDaemonProcess { name: wallet_daemon_name.clone(), json_rpc_port, - indexer_jrpc_port, + indexer_api_port, temp_path_dir: base_dir, shutdown, }; diff --git a/integration_tests/tests/steps/indexer.rs b/integration_tests/tests/steps/indexer.rs index f90318d892..c77a182ab4 100644 --- a/integration_tests/tests/steps/indexer.rs +++ b/integration_tests/tests/steps/indexer.rs @@ -29,7 +29,7 @@ async fn given_validator_connects_to_other_vns(world: &mut TariWorld, name: Stri ) }); - let mut cli = indexer.get_jrpc_indexer_client(); + let mut cli = indexer.get_indexer_client(); for (pk, addr) in details { if let Err(err) = cli .add_peer(AddPeerRequest { @@ -49,7 +49,7 @@ async fn given_validator_connects_to_other_vns(world: &mut TariWorld, name: Stri #[then(expr = "indexer {word} has scanned to at least height {int}")] pub async fn indexer_has_scanned_to_at_least_height(world: &mut TariWorld, name: String, block_height: u64) { let indexer = world.get_indexer(&name); - let mut client = indexer.get_jrpc_indexer_client(); + let mut client = indexer.get_indexer_client(); let mut last_block_height = 0; let mut remaining = 10; loop { @@ -250,7 +250,7 @@ async fn i_wait_for_the_indexer_to_sync_with_the_network(world: &mut TariWorld, let indexer = world.get_indexer(&indexer_name); assert!(!indexer.handle.is_finished(), "Indexer {} is not running", indexer_name); - let mut client = indexer.get_jrpc_indexer_client(); + let mut client = indexer.get_indexer_client(); let mut remaining_attempts = 60; loop { if remaining_attempts == 0 { diff --git a/integration_tests/tests/steps/validator_node.rs b/integration_tests/tests/steps/validator_node.rs index b609b292c6..a8b7637446 100644 --- a/integration_tests/tests/steps/validator_node.rs +++ b/integration_tests/tests/steps/validator_node.rs @@ -62,7 +62,7 @@ async fn spawn_seed_node( .unwrap(); } for indexer in world.indexers.values() { - let mut client = indexer.get_jrpc_indexer_client(); + let mut client = indexer.get_indexer_client(); client .add_peer(tari_indexer_client::types::AddPeerRequest { public_key: ident.public_key, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 691246bbe3..d1e1d1902d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,6 +50,9 @@ importers: '@tanstack/react-query-devtools': specifier: ^5.84.2 version: 5.85.5(@tanstack/react-query@5.81.2(react@19.1.0))(react@19.1.0) + '@tari-project/indexer-client': + specifier: link:../../../clients/javascript/indexer_client + version: link:../../../clients/javascript/indexer_client '@tari-project/typescript-bindings': specifier: link:../../../bindings version: link:../../../bindings @@ -310,6 +313,19 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/debug@4.1.12)(@types/node@22.14.0)(jiti@2.4.2)(lightningcss@1.30.1)(msw@2.10.2(@types/node@22.14.0)(typescript@5.8.3)) + clients/javascript/indexer_client: + dependencies: + '@tari-project/typescript-bindings': + specifier: workspace:^ + version: link:../../../bindings + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 22.14.0 + typescript: + specifier: 'catalog:' + version: 5.8.3 + clients/javascript/wallet_daemon_client: dependencies: '@tari-project/typescript-bindings': diff --git a/scripts/build_webuis.sh b/scripts/build_webuis.sh index d6e5bdddac..66bb92ad12 100755 --- a/scripts/build_webuis.sh +++ b/scripts/build_webuis.sh @@ -90,6 +90,9 @@ set +e echo "Building Wallet client..." build clients/javascript/wallet_daemon_client +echo "Building Indexer client..." +build clients/javascript/indexer_client + # Build webuis echo "Building Validator Node Web UI..." build applications/tari_validator_node/web_ui @@ -97,6 +100,5 @@ build applications/tari_validator_node/web_ui echo "Building Wallet Web UI..." build applications/tari_walletd/web_ui - echo "Building Indexer Web UI..." build applications/tari_indexer/web_ui diff --git a/utilities/tariswap_test_bench/Cargo.toml b/utilities/tariswap_test_bench/Cargo.toml index bee1a4d4f0..c56af46632 100644 --- a/utilities/tariswap_test_bench/Cargo.toml +++ b/utilities/tariswap_test_bench/Cargo.toml @@ -9,7 +9,7 @@ license.workspace = true [dependencies] tari_crypto = { workspace = true } tari_ootle_wallet_sdk = { workspace = true } -tari_ootle_wallet_sdk_services = { workspace = true, features = ["indexer_jrpc"] } +tari_ootle_wallet_sdk_services = { workspace = true, features = ["indexer_client"] } tari_ootle_common_types = { workspace = true } tari_ootle_wallet_storage_sqlite = { workspace = true } tari_engine_types = { workspace = true } diff --git a/utilities/tariswap_test_bench/src/cli.rs b/utilities/tariswap_test_bench/src/cli.rs index 68f865faa4..36db7e050a 100644 --- a/utilities/tariswap_test_bench/src/cli.rs +++ b/utilities/tariswap_test_bench/src/cli.rs @@ -27,12 +27,7 @@ impl Cli { pub struct CommonArgs { #[clap(long, short = 'd', alias = "db", default_value = "data/tariswap-test-bench.sqlite")] pub db_path: PathBuf, - #[clap( - long, - short = 'i', - alias = "indexer", - default_value = "http://localhost:18300/json_rpc" - )] + #[clap(long, short = 'i', alias = "indexer", default_value = "http://localhost:18300")] pub indexer_url: Url, #[clap(long, short = 'v', alias = "vn", default_value = "http://localhost:18200/json_rpc")] pub validator_node_url: Url, diff --git a/utilities/tariswap_test_bench/src/runner.rs b/utilities/tariswap_test_bench/src/runner.rs index ae0375bdba..f31ef8b864 100644 --- a/utilities/tariswap_test_bench/src/runner.rs +++ b/utilities/tariswap_test_bench/src/runner.rs @@ -8,7 +8,7 @@ use tari_crypto::tari_utilities::SafePassword; use tari_engine_types::commit_result::FinalizeResult; use tari_ootle_common_types::Network; use tari_ootle_wallet_sdk::{cipher_seed::CipherSeedRestore, WalletSdk as Sdk, WalletSdkConfig}; -use tari_ootle_wallet_sdk_services::indexer_jrpc::IndexerJsonRpcNetworkInterface; +use tari_ootle_wallet_sdk_services::indexer_rest_api::IndexerRestApiNetworkInterface; use tari_ootle_wallet_storage_sqlite::SqliteWalletStore; use tari_transaction::{Transaction, TransactionBuilder, TransactionId}; use tari_validator_node_client::types::TemplateMetadata; @@ -17,7 +17,7 @@ use url::Url; use crate::{cli::CommonArgs, stats::Stats, templates::get_templates}; -type WalletSdk = Sdk; +type WalletSdk = Sdk; pub struct Runner { pub(crate) sdk: WalletSdk, pub(crate) _cli: CommonArgs, @@ -119,7 +119,7 @@ fn initialize_wallet_sdk>(db_path: P, indexer_url: Url) -> Result network: Network::LocalNet, override_keyring_password: Some(SafePassword::from_str("N3Va g0nn4 gu355").unwrap()), }; - let indexer = IndexerJsonRpcNetworkInterface::new(indexer_url); + let indexer = IndexerRestApiNetworkInterface::new(indexer_url); let mut sdk = WalletSdk::initialize(store, indexer, sdk_config)?; sdk.initialize_cipher_seed(CipherSeedRestore::CreateNewIfRequired)?; Ok(sdk)