diff --git a/cumulus/test/service/src/lib.rs b/cumulus/test/service/src/lib.rs index 1031f9c93025..55366e593865 100644 --- a/cumulus/test/service/src/lib.rs +++ b/cumulus/test/service/src/lib.rs @@ -848,7 +848,7 @@ pub fn node_config( transaction_pool: Default::default(), network: network_config, keystore: KeystoreConfig::InMemory, - database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 }, + database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128, transaction_column_path: None }, trie_cache_maximum_size: Some(64 * 1024 * 1024), warm_up_trie_cache: None, state_pruning: Some(PruningMode::ArchiveAll), diff --git a/polkadot/node/test/service/src/lib.rs b/polkadot/node/test/service/src/lib.rs index 8141e0897398..adabb91df3c9 100644 --- a/polkadot/node/test/service/src/lib.rs +++ b/polkadot/node/test/service/src/lib.rs @@ -185,7 +185,7 @@ pub fn node_config( transaction_pool: Default::default(), network: network_config, keystore: KeystoreConfig::InMemory, - database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 }, + database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128, transaction_column_path: None }, trie_cache_maximum_size: Some(64 * 1024 * 1024), warm_up_trie_cache: None, state_pruning: Default::default(), diff --git a/substrate/bin/node/cli/benches/block_production.rs b/substrate/bin/node/cli/benches/block_production.rs index 13f06724c38a..5546af61e3fd 100644 --- a/substrate/bin/node/cli/benches/block_production.rs +++ b/substrate/bin/node/cli/benches/block_production.rs @@ -70,7 +70,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase { transaction_pool: Default::default(), network: network_config, keystore: KeystoreConfig::InMemory, - database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 }, + database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128, transaction_column_path: None }, trie_cache_maximum_size: Some(64 * 1024 * 1024), warm_up_trie_cache: None, state_pruning: Some(PruningMode::ArchiveAll), diff --git a/substrate/bin/node/cli/benches/transaction_pool.rs b/substrate/bin/node/cli/benches/transaction_pool.rs index cd7054326e7d..2523c54745a3 100644 --- a/substrate/bin/node/cli/benches/transaction_pool.rs +++ b/substrate/bin/node/cli/benches/transaction_pool.rs @@ -61,7 +61,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase { transaction_pool: TransactionPoolOptions::new_for_benchmarks(), network: network_config, keystore: KeystoreConfig::InMemory, - database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 }, + database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128, transaction_column_path: None }, trie_cache_maximum_size: Some(64 * 1024 * 1024), warm_up_trie_cache: None, state_pruning: Some(PruningMode::ArchiveAll), diff --git a/substrate/bin/node/testing/src/bench.rs b/substrate/bin/node/testing/src/bench.rs index da067c2d6cba..d11b32a2d997 100644 --- a/substrate/bin/node/testing/src/bench.rs +++ b/substrate/bin/node/testing/src/bench.rs @@ -222,7 +222,7 @@ pub enum DatabaseType { impl DatabaseType { fn into_settings(self, path: PathBuf) -> sc_client_db::DatabaseSource { match self { - Self::RocksDb => sc_client_db::DatabaseSource::RocksDb { path, cache_size: 512 }, + Self::RocksDb => sc_client_db::DatabaseSource::RocksDb { path, cache_size: 512, transaction_column_path: None }, Self::ParityDb => sc_client_db::DatabaseSource::ParityDb { path }, } } diff --git a/substrate/client/cli/src/config.rs b/substrate/client/cli/src/config.rs index 0af508aeed6d..c0fd731ea564 100644 --- a/substrate/client/cli/src/config.rs +++ b/substrate/client/cli/src/config.rs @@ -231,7 +231,13 @@ pub trait CliConfiguration: Sized { let paritydb_path = base_path.join("paritydb").join(role_dir); Ok(match database { #[cfg(feature = "rocksdb")] - Database::RocksDb => DatabaseSource::RocksDb { path: rocksdb_path, cache_size }, + Database::RocksDb => DatabaseSource::RocksDb { + path: rocksdb_path, + cache_size, + transaction_column_path: self + .database_params() + .and_then(|p| p.transaction_storage_path().map(|p| p.to_path_buf())), + }, Database::ParityDb => DatabaseSource::ParityDb { path: paritydb_path }, Database::ParityDbDeprecated => { eprintln!( diff --git a/substrate/client/cli/src/params/database_params.rs b/substrate/client/cli/src/params/database_params.rs index cbc602cb877b..b192cf59603a 100644 --- a/substrate/client/cli/src/params/database_params.rs +++ b/substrate/client/cli/src/params/database_params.rs @@ -18,6 +18,7 @@ use crate::arg_enums::Database; use clap::Args; +use std::path::{Path, PathBuf}; /// Parameters for database #[derive(Debug, Clone, PartialEq, Args)] @@ -29,6 +30,14 @@ pub struct DatabaseParams { /// Limit the memory the database cache can use. #[arg(long = "db-cache", value_name = "MiB")] pub database_cache_size: Option, + + /// Directory for the indexed-transaction column when using the RocksDB backend. + /// + /// Places that column's data under this path (e.g. a separate, cheaper volume) instead + /// of the main database directory. Must stay consistent across restarts. Only used by + /// chains that index transaction data (transaction storage). + #[arg(long, value_name = "PATH")] + pub transaction_storage_path: Option, } impl DatabaseParams { @@ -41,4 +50,9 @@ impl DatabaseParams { pub fn database_cache_size(&self) -> Option { self.database_cache_size } + + /// Directory for the indexed-transaction column (RocksDB only). + pub fn transaction_storage_path(&self) -> Option<&Path> { + self.transaction_storage_path.as_deref() + } } diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs index b0da86455f01..40aab6d82d68 100644 --- a/substrate/client/db/src/lib.rs +++ b/substrate/client/db/src/lib.rs @@ -395,6 +395,12 @@ pub enum DatabaseSource { path: PathBuf, /// Cache size in MiB. cache_size: usize, + /// Optional directory for the `TRANSACTION` column (indexed transaction data). + /// + /// When set, that column's SST files are written under this path, so the bulk, cold + /// data of transaction-storage chains can live on a separate (cheaper) volume. Must + /// stay consistent across restarts of the same database. + transaction_column_path: Option, }, /// Load a ParityDb database from a given path. @@ -4549,6 +4555,94 @@ pub(crate) mod tests { assert_eq!(bc.indexed_transaction(x1_hash).unwrap(), None); } + #[test] + fn transaction_column_on_override_path() { + let db_dir = tempfile::tempdir().unwrap(); + let cold_dir = tempfile::tempdir().unwrap(); + let cold_col_dir = cold_dir.path().join("col11"); + + fn sst_files(dir: &std::path::Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + if let Ok(entries) = std::fs::read_dir(&d) { + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + } else if p.extension().map_or(false, |x| x == "sst") { + out.push(p.file_name().unwrap().to_string_lossy().into_owned()); + } + } + } + } + out + } + + const PAYLOADS: u64 = 30; + let payload = |i: u64| -> Vec { + let mut v = vec![0u8; 256 * 1024]; + v[..8].copy_from_slice(&i.to_le_bytes()); + v + }; + let payload_hash = |i: u64| H256::from_low_u64_le(i + 1); + + let open_backend = || { + Backend::::new_test_with_tx_storage_source( + BlocksPruning::KeepAll, + 0, + DatabaseSource::RocksDb { + path: db_dir.path().join("db"), + cache_size: 128, + transaction_column_path: Some(cold_col_dir.clone()), + }, + Default::default(), + ) + }; + + let backend = open_backend(); + let mut parent = + insert_block(&backend, 0, Default::default(), None, Default::default(), vec![], None) + .unwrap(); + for i in 0..PAYLOADS { + let mut renew_payloads = std::collections::HashMap::new(); + renew_payloads.insert(payload_hash(i), payload(i)); + let ops = vec![IndexOperation::Renew { + extrinsic: 0, + hash: payload_hash(i).as_ref().to_vec(), + }]; + parent = insert_block_with_synthetic_ops( + &backend, + i + 1, + parent, + Default::default(), + vec![UncheckedXt::new_transaction(i.into(), ())], + vec![], + ops, + renew_payloads, + ) + .unwrap(); + } + let bc = backend.blockchain(); + for i in 0..PAYLOADS { + assert_eq!(bc.indexed_transaction(payload_hash(i)).unwrap().unwrap(), payload(i)); + } + drop(backend); + + let cold_ssts = sst_files(&cold_col_dir); + assert!(!cold_ssts.is_empty(), "no SST files under override dir {:?}", cold_col_dir); + let main_ssts = sst_files(&db_dir.path().join("db")); + for f in &cold_ssts { + assert!(!main_ssts.contains(f), "SST {} present in BOTH main and override dirs", f); + } + + let backend = open_backend(); + let bc = backend.blockchain(); + for i in 0..PAYLOADS { + assert_eq!(bc.indexed_transaction(payload_hash(i)).unwrap().unwrap(), payload(i)); + } + } + #[test] fn index_invalid_size() { let backend = Backend::::new_test_with_tx_storage(BlocksPruning::Some(1), 10); diff --git a/substrate/client/db/src/upgrade.rs b/substrate/client/db/src/upgrade.rs index f8833a337b28..2bac49a31106 100644 --- a/substrate/client/db/src/upgrade.rs +++ b/substrate/client/db/src/upgrade.rs @@ -206,7 +206,7 @@ mod tests { fn open_database(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> { crate::utils::open_database::( - &DatabaseSource::RocksDb { path: db_path.to_owned(), cache_size: 128 }, + &DatabaseSource::RocksDb { path: db_path.to_owned(), cache_size: 128, transaction_column_path: None }, db_type, true, ) diff --git a/substrate/client/db/src/utils.rs b/substrate/client/db/src/utils.rs index c048119ee622..fc17f96bb974 100644 --- a/substrate/client/db/src/utils.rs +++ b/substrate/client/db/src/utils.rs @@ -202,8 +202,14 @@ fn open_database_at( let db: Arc> = match &db_source { DatabaseSource::ParityDb { path } => open_parity_db::(path, db_type, create)?, #[cfg(feature = "rocksdb")] - DatabaseSource::RocksDb { path, cache_size } => { - open_kvdb_rocksdb::(path, db_type, create, *cache_size)? + DatabaseSource::RocksDb { path, cache_size, transaction_column_path } => { + open_kvdb_rocksdb::( + path, + db_type, + create, + *cache_size, + transaction_column_path.as_deref(), + )? }, DatabaseSource::Custom { db, require_create_flag } => { if *require_create_flag && !create { @@ -213,7 +219,7 @@ fn open_database_at( }, DatabaseSource::Auto { paritydb_path, rocksdb_path, cache_size } => { // check if rocksdb exists first, if not, open paritydb - match open_kvdb_rocksdb::(rocksdb_path, db_type, false, *cache_size) { + match open_kvdb_rocksdb::(rocksdb_path, db_type, false, *cache_size, None) { Ok(db) => db, Err(OpenDbError::NotEnabled(_)) | Err(OpenDbError::DoesNotExist) => { open_parity_db::(paritydb_path, db_type, create)? @@ -310,6 +316,7 @@ fn open_kvdb_rocksdb( db_type: DatabaseType, create: bool, cache_size: usize, + transaction_column_path: Option<&Path>, ) -> OpenDbResult { // first upgrade database to required version match crate::upgrade::upgrade_db::(path, db_type) { @@ -323,6 +330,10 @@ fn open_kvdb_rocksdb( let mut db_config = kvdb_rocksdb::DatabaseConfig::with_columns(NUM_COLUMNS); db_config.create_if_missing = create; + if let Some(cold) = transaction_column_path { + db_config.columns[crate::columns::TRANSACTION as usize].path = Some(cold.to_path_buf()); + } + let mut memory_budget = std::collections::HashMap::new(); match db_type { DatabaseType::Full => { @@ -346,9 +357,25 @@ fn open_kvdb_rocksdb( ); }, } - db_config.memory_budget = memory_budget; + // kvdb-rocksdb master moved the memory budget into per-column `ColumnConfig`. + for (col, budget) in memory_budget { + db_config.columns[col as usize].memory_budget = Some(budget); + } - let db = kvdb_rocksdb::Database::open(&db_config, path)?; + let db = kvdb_rocksdb::Database::open(&db_config, path).map_err(|err| { + if transaction_column_path.is_some() { + io::Error::new( + err.kind(), + format!( + "error opening database with a transaction column path override; the \ + override must match the path used when the database was last opened: {}", + err + ), + ) + } else { + err + } + })?; // write database version only after the database is successfully opened crate::upgrade::update_version(path)?; Ok(sp_database::as_rocksdb_database(db)) @@ -360,6 +387,7 @@ fn open_kvdb_rocksdb( _db_type: DatabaseType, _create: bool, _cache_size: usize, + _transaction_column_path: Option<&Path>, ) -> OpenDbResult { Err(OpenDbError::NotEnabled("with-kvdb-rocksdb")) } @@ -658,7 +686,7 @@ mod tests { check_dir_for_db_type( DatabaseType::Full, - DatabaseSource::RocksDb { path: PathBuf::new(), cache_size: 128 }, + DatabaseSource::RocksDb { path: PathBuf::new(), cache_size: 128, transaction_column_path: None }, "db_version", ); @@ -673,7 +701,7 @@ mod tests { let base_path = tempfile::TempDir::new().unwrap(); let old_db_path = base_path.path().join("chains/dev/db"); - let source = DatabaseSource::RocksDb { path: old_db_path.clone(), cache_size: 128 }; + let source = DatabaseSource::RocksDb { path: old_db_path.clone(), cache_size: 128, transaction_column_path: None }; { let db_res = open_database::(&source, DatabaseType::Full, true); assert!(db_res.is_ok(), "New database should be created."); @@ -752,7 +780,7 @@ mod tests { // it should fail to open existing auto (pairtydb) database { let db_res = open_database::( - &DatabaseSource::RocksDb { path: rocksdb_path, cache_size: 128 }, + &DatabaseSource::RocksDb { path: rocksdb_path, cache_size: 128, transaction_column_path: None }, DatabaseType::Full, true, ); @@ -778,7 +806,7 @@ mod tests { let paritydb_path = db_path.join("paritydb"); let rocksdb_path = db_path.join("rocksdb_path"); - let source = DatabaseSource::RocksDb { path: rocksdb_path.clone(), cache_size: 128 }; + let source = DatabaseSource::RocksDb { path: rocksdb_path.clone(), cache_size: 128, transaction_column_path: None }; // it should create new rocksdb database { @@ -813,7 +841,7 @@ mod tests { // it should reopen existing auto (pairtydb) database { let db_res = open_database::( - &DatabaseSource::RocksDb { path: rocksdb_path, cache_size: 128 }, + &DatabaseSource::RocksDb { path: rocksdb_path, cache_size: 128, transaction_column_path: None }, DatabaseType::Full, true, ); @@ -846,7 +874,7 @@ mod tests { // it should fail to open existing pairtydb database { let db_res = open_database::( - &DatabaseSource::RocksDb { path: rocksdb_path.clone(), cache_size: 128 }, + &DatabaseSource::RocksDb { path: rocksdb_path.clone(), cache_size: 128, transaction_column_path: None }, DatabaseType::Full, true, ); diff --git a/substrate/client/service/test/src/client/mod.rs b/substrate/client/service/test/src/client/mod.rs index 6e3dbb57b4df..973809581c10 100644 --- a/substrate/client/service/test/src/client/mod.rs +++ b/substrate/client/service/test/src/client/mod.rs @@ -1496,7 +1496,7 @@ fn doesnt_import_blocks_that_revert_finality() { state_pruning: Some(PruningMode::ArchiveAll), blocks_pruning: BlocksPruning::KeepAll, pruning_filters: Default::default(), - source: DatabaseSource::RocksDb { path: tmp.path().into(), cache_size: 1024 }, + source: DatabaseSource::RocksDb { path: tmp.path().into(), cache_size: 1024, transaction_column_path: None }, metrics_registry: None, }, u64::MAX, @@ -1778,7 +1778,7 @@ fn returns_status_for_pruned_blocks() { state_pruning: Some(PruningMode::blocks_pruning(1)), blocks_pruning: BlocksPruning::KeepFinalized, pruning_filters: Default::default(), - source: DatabaseSource::RocksDb { path: tmp.path().into(), cache_size: 1024 }, + source: DatabaseSource::RocksDb { path: tmp.path().into(), cache_size: 1024, transaction_column_path: None }, metrics_registry: None, }, u64::MAX, diff --git a/substrate/client/service/test/src/lib.rs b/substrate/client/service/test/src/lib.rs index 94c1426d03f8..7e7716c586e9 100644 --- a/substrate/client/service/test/src/lib.rs +++ b/substrate/client/service/test/src/lib.rs @@ -233,7 +233,7 @@ fn node_config( transaction_pool: Default::default(), network: network_config, keystore: KeystoreConfig::Path { path: root.join("key"), password: None }, - database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 }, + database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128, transaction_column_path: None }, trie_cache_maximum_size: Some(16 * 1024 * 1024), warm_up_trie_cache: None, state_pruning: Default::default(),