sc-client-db: optional path override for the TRANSACTION column

cumulus/test/service/src/lib.rs

@@ -848,7 +848,7 @@ pub fn node_config(
 		transaction_pool: Default::default(),
 		network: network_config,
 		keystore: KeystoreConfig::InMemory,
-		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 },
+		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128, transaction_column_path: None },
 		trie_cache_maximum_size: Some(64 * 1024 * 1024),
 		warm_up_trie_cache: None,
 		state_pruning: Some(PruningMode::ArchiveAll),

polkadot/node/test/service/src/lib.rs

@@ -185,7 +185,7 @@ pub fn node_config(
 		transaction_pool: Default::default(),
 		network: network_config,
 		keystore: KeystoreConfig::InMemory,
-		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 },
+		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128, transaction_column_path: None },
 		trie_cache_maximum_size: Some(64 * 1024 * 1024),
 		warm_up_trie_cache: None,
 		state_pruning: Default::default(),

substrate/bin/node/cli/benches/block_production.rs

@@ -70,7 +70,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
 		transaction_pool: Default::default(),
 		network: network_config,
 		keystore: KeystoreConfig::InMemory,
-		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 },
+		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128, transaction_column_path: None },
 		trie_cache_maximum_size: Some(64 * 1024 * 1024),
 		warm_up_trie_cache: None,
 		state_pruning: Some(PruningMode::ArchiveAll),

substrate/bin/node/cli/benches/transaction_pool.rs

@@ -61,7 +61,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
 		transaction_pool: TransactionPoolOptions::new_for_benchmarks(),
 		network: network_config,
 		keystore: KeystoreConfig::InMemory,
-		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 },
+		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128, transaction_column_path: None },
 		trie_cache_maximum_size: Some(64 * 1024 * 1024),
 		warm_up_trie_cache: None,
 		state_pruning: Some(PruningMode::ArchiveAll),

substrate/bin/node/testing/src/bench.rs

@@ -222,7 +222,7 @@ pub enum DatabaseType {
 impl DatabaseType {
 	fn into_settings(self, path: PathBuf) -> sc_client_db::DatabaseSource {
 		match self {
-			Self::RocksDb => sc_client_db::DatabaseSource::RocksDb { path, cache_size: 512 },
+			Self::RocksDb => sc_client_db::DatabaseSource::RocksDb { path, cache_size: 512, transaction_column_path: None },
 			Self::ParityDb => sc_client_db::DatabaseSource::ParityDb { path },
 		}
 	}

substrate/client/cli/src/config.rs

@@ -231,7 +231,13 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 		let paritydb_path = base_path.join("paritydb").join(role_dir);
 		Ok(match database {
 			#[cfg(feature = "rocksdb")]
-			Database::RocksDb => DatabaseSource::RocksDb { path: rocksdb_path, cache_size },
+			Database::RocksDb => DatabaseSource::RocksDb {
+				path: rocksdb_path,
+				cache_size,
+				transaction_column_path: self
+					.database_params()
+					.and_then(|p| p.transaction_storage_path().map(|p| p.to_path_buf())),
+			},
 			Database::ParityDb => DatabaseSource::ParityDb { path: paritydb_path },
 			Database::ParityDbDeprecated => {
 				eprintln!(

substrate/client/cli/src/params/database_params.rs

@@ -18,6 +18,7 @@
 
 use crate::arg_enums::Database;
 use clap::Args;
+use std::path::{Path, PathBuf};
 
 /// Parameters for database
 #[derive(Debug, Clone, PartialEq, Args)]
@@ -29,6 +30,14 @@ pub struct DatabaseParams {
 	/// Limit the memory the database cache can use.
 	#[arg(long = "db-cache", value_name = "MiB")]
 	pub database_cache_size: Option<usize>,
+
+	/// Directory for the indexed-transaction column when using the RocksDB backend.
+	///
+	/// Places that column's data under this path (e.g. a separate, cheaper volume) instead
+	/// of the main database directory. Must stay consistent across restarts. Only used by
+	/// chains that index transaction data (transaction storage).
+	#[arg(long, value_name = "PATH")]
+	pub transaction_storage_path: Option<PathBuf>,
 }
 
 impl DatabaseParams {
@@ -41,4 +50,9 @@ impl DatabaseParams {
 	pub fn database_cache_size(&self) -> Option<usize> {
 		self.database_cache_size
 	}
+
+	/// Directory for the indexed-transaction column (RocksDB only).
+	pub fn transaction_storage_path(&self) -> Option<&Path> {
+		self.transaction_storage_path.as_deref()
+	}
 }

substrate/client/db/src/lib.rs

@@ -395,6 +395,12 @@ pub enum DatabaseSource {
 		path: PathBuf,
 		/// Cache size in MiB.
 		cache_size: usize,
+		/// Optional directory for the `TRANSACTION` column (indexed transaction data).
+		///
+		/// When set, that column's SST files are written under this path, so the bulk, cold
+		/// data of transaction-storage chains can live on a separate (cheaper) volume. Must
+		/// stay consistent across restarts of the same database.
+		transaction_column_path: Option<PathBuf>,
 	},
 
 	/// Load a ParityDb database from a given path.
@@ -4549,6 +4555,94 @@ pub(crate) mod tests {
 		assert_eq!(bc.indexed_transaction(x1_hash).unwrap(), None);
 	}
 
+	#[test]
+	fn transaction_column_on_override_path() {
+		let db_dir = tempfile::tempdir().unwrap();
+		let cold_dir = tempfile::tempdir().unwrap();
+		let cold_col_dir = cold_dir.path().join("col11");
+
+		fn sst_files(dir: &std::path::Path) -> Vec<String> {
+			let mut out = Vec::new();
+			let mut stack = vec![dir.to_path_buf()];
+			while let Some(d) = stack.pop() {
+				if let Ok(entries) = std::fs::read_dir(&d) {
+					for e in entries.flatten() {
+						let p = e.path();
+						if p.is_dir() {
+							stack.push(p);
+						} else if p.extension().map_or(false, |x| x == "sst") {
+							out.push(p.file_name().unwrap().to_string_lossy().into_owned());
+						}
+					}
+				}
+			}
+			out
+		}
+
+		const PAYLOADS: u64 = 30;
+		let payload = |i: u64| -> Vec<u8> {
+			let mut v = vec![0u8; 256 * 1024];
+			v[..8].copy_from_slice(&i.to_le_bytes());
+			v
+		};
+		let payload_hash = |i: u64| H256::from_low_u64_le(i + 1);
+
+		let open_backend = || {
+			Backend::<Block>::new_test_with_tx_storage_source(
+				BlocksPruning::KeepAll,
+				0,
+				DatabaseSource::RocksDb {
+					path: db_dir.path().join("db"),
+					cache_size: 128,
+					transaction_column_path: Some(cold_col_dir.clone()),
+				},
+				Default::default(),
+			)
+		};
+
+		let backend = open_backend();
+		let mut parent =
+			insert_block(&backend, 0, Default::default(), None, Default::default(), vec![], None)
+				.unwrap();
+		for i in 0..PAYLOADS {
+			let mut renew_payloads = std::collections::HashMap::new();
+			renew_payloads.insert(payload_hash(i), payload(i));
+			let ops = vec![IndexOperation::Renew {
+				extrinsic: 0,
+				hash: payload_hash(i).as_ref().to_vec(),
+			}];
+			parent = insert_block_with_synthetic_ops(
+				&backend,
+				i + 1,
+				parent,
+				Default::default(),
+				vec![UncheckedXt::new_transaction(i.into(), ())],
+				vec![],
+				ops,
+				renew_payloads,
+			)
+			.unwrap();
+		}
+		let bc = backend.blockchain();
+		for i in 0..PAYLOADS {
+			assert_eq!(bc.indexed_transaction(payload_hash(i)).unwrap().unwrap(), payload(i));
+		}
+		drop(backend);
+
+		let cold_ssts = sst_files(&cold_col_dir);
+		assert!(!cold_ssts.is_empty(), "no SST files under override dir {:?}", cold_col_dir);
+		let main_ssts = sst_files(&db_dir.path().join("db"));
+		for f in &cold_ssts {
+			assert!(!main_ssts.contains(f), "SST {} present in BOTH main and override dirs", f);
+		}
+
+		let backend = open_backend();
+		let bc = backend.blockchain();
+		for i in 0..PAYLOADS {
+			assert_eq!(bc.indexed_transaction(payload_hash(i)).unwrap().unwrap(), payload(i));
+		}
+	}
+
 	#[test]
 	fn index_invalid_size() {
 		let backend = Backend::<Block>::new_test_with_tx_storage(BlocksPruning::Some(1), 10);

substrate/client/db/src/upgrade.rs

@@ -206,7 +206,7 @@ mod tests {
 
 	fn open_database(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
 		crate::utils::open_database::<Block>(
-			&DatabaseSource::RocksDb { path: db_path.to_owned(), cache_size: 128 },
+			&DatabaseSource::RocksDb { path: db_path.to_owned(), cache_size: 128, transaction_column_path: None },
 			db_type,
 			true,
 		)

substrate/client/db/src/utils.rs

@@ -202,8 +202,14 @@ fn open_database_at<Block: BlockT>(
 	let db: Arc<dyn Database<DbHash>> = match &db_source {
 		DatabaseSource::ParityDb { path } => open_parity_db::<Block>(path, db_type, create)?,
 		#[cfg(feature = "rocksdb")]
-		DatabaseSource::RocksDb { path, cache_size } => {
-			open_kvdb_rocksdb::<Block>(path, db_type, create, *cache_size)?
+		DatabaseSource::RocksDb { path, cache_size, transaction_column_path } => {
+			open_kvdb_rocksdb::<Block>(
+				path,
+				db_type,
+				create,
+				*cache_size,
+				transaction_column_path.as_deref(),
+			)?
 		},
 		DatabaseSource::Custom { db, require_create_flag } => {
 			if *require_create_flag && !create {
@@ -213,7 +219,7 @@ fn open_database_at<Block: BlockT>(
 		},
 		DatabaseSource::Auto { paritydb_path, rocksdb_path, cache_size } => {
 			// check if rocksdb exists first, if not, open paritydb
-			match open_kvdb_rocksdb::<Block>(rocksdb_path, db_type, false, *cache_size) {
+			match open_kvdb_rocksdb::<Block>(rocksdb_path, db_type, false, *cache_size, None) {
 				Ok(db) => db,
 				Err(OpenDbError::NotEnabled(_)) | Err(OpenDbError::DoesNotExist) => {
 					open_parity_db::<Block>(paritydb_path, db_type, create)?
@@ -310,6 +316,7 @@ fn open_kvdb_rocksdb<Block: BlockT>(
 	db_type: DatabaseType,
 	create: bool,
 	cache_size: usize,
+	transaction_column_path: Option<&Path>,
 ) -> OpenDbResult {
 	// first upgrade database to required version
 	match crate::upgrade::upgrade_db::<Block>(path, db_type) {
@@ -323,6 +330,10 @@ fn open_kvdb_rocksdb<Block: BlockT>(
 	let mut db_config = kvdb_rocksdb::DatabaseConfig::with_columns(NUM_COLUMNS);
 	db_config.create_if_missing = create;
 
+	if let Some(cold) = transaction_column_path {
+		db_config.columns[crate::columns::TRANSACTION as usize].path = Some(cold.to_path_buf());
+	}
+
 	let mut memory_budget = std::collections::HashMap::new();
 	match db_type {
 		DatabaseType::Full => {
@@ -346,9 +357,25 @@ fn open_kvdb_rocksdb<Block: BlockT>(
 			);
 		},
 	}
-	db_config.memory_budget = memory_budget;
+	// kvdb-rocksdb master moved the memory budget into per-column `ColumnConfig`.
+	for (col, budget) in memory_budget {
+		db_config.columns[col as usize].memory_budget = Some(budget);
+	}
 
-	let db = kvdb_rocksdb::Database::open(&db_config, path)?;
+	let db = kvdb_rocksdb::Database::open(&db_config, path).map_err(|err| {
+		if transaction_column_path.is_some() {
+			io::Error::new(
+				err.kind(),
+				format!(
+					"error opening database with a transaction column path override; the \
+					 override must match the path used when the database was last opened: {}",
+					err
+				),
+			)
+		} else {
+			err
+		}
+	})?;
 	// write database version only after the database is successfully opened
 	crate::upgrade::update_version(path)?;
 	Ok(sp_database::as_rocksdb_database(db))
@@ -360,6 +387,7 @@ fn open_kvdb_rocksdb<Block: BlockT>(
 	_db_type: DatabaseType,
 	_create: bool,
 	_cache_size: usize,
+	_transaction_column_path: Option<&Path>,
 ) -> OpenDbResult {
 	Err(OpenDbError::NotEnabled("with-kvdb-rocksdb"))
 }
@@ -658,7 +686,7 @@ mod tests {
 
 		check_dir_for_db_type(
 			DatabaseType::Full,
-			DatabaseSource::RocksDb { path: PathBuf::new(), cache_size: 128 },
+			DatabaseSource::RocksDb { path: PathBuf::new(), cache_size: 128, transaction_column_path: None },
 			"db_version",
 		);
 
@@ -673,7 +701,7 @@ mod tests {
 			let base_path = tempfile::TempDir::new().unwrap();
 			let old_db_path = base_path.path().join("chains/dev/db");
 
-			let source = DatabaseSource::RocksDb { path: old_db_path.clone(), cache_size: 128 };
+			let source = DatabaseSource::RocksDb { path: old_db_path.clone(), cache_size: 128, transaction_column_path: None };
 			{
 				let db_res = open_database::<Block>(&source, DatabaseType::Full, true);
 				assert!(db_res.is_ok(), "New database should be created.");
@@ -752,7 +780,7 @@ mod tests {
 		// it should fail to open existing auto (pairtydb) database
 		{
 			let db_res = open_database::<Block>(
-				&DatabaseSource::RocksDb { path: rocksdb_path, cache_size: 128 },
+				&DatabaseSource::RocksDb { path: rocksdb_path, cache_size: 128, transaction_column_path: None },
 				DatabaseType::Full,
 				true,
 			);
@@ -778,7 +806,7 @@ mod tests {
 		let paritydb_path = db_path.join("paritydb");
 		let rocksdb_path = db_path.join("rocksdb_path");
 
-		let source = DatabaseSource::RocksDb { path: rocksdb_path.clone(), cache_size: 128 };
+		let source = DatabaseSource::RocksDb { path: rocksdb_path.clone(), cache_size: 128, transaction_column_path: None };
 
 		// it should create new rocksdb database
 		{
@@ -813,7 +841,7 @@ mod tests {
 		// it should reopen existing auto (pairtydb) database
 		{
 			let db_res = open_database::<Block>(
-				&DatabaseSource::RocksDb { path: rocksdb_path, cache_size: 128 },
+				&DatabaseSource::RocksDb { path: rocksdb_path, cache_size: 128, transaction_column_path: None },
 				DatabaseType::Full,
 				true,
 			);
@@ -846,7 +874,7 @@ mod tests {
 		// it should fail to open existing pairtydb database
 		{
 			let db_res = open_database::<Block>(
-				&DatabaseSource::RocksDb { path: rocksdb_path.clone(), cache_size: 128 },
+				&DatabaseSource::RocksDb { path: rocksdb_path.clone(), cache_size: 128, transaction_column_path: None },
 				DatabaseType::Full,
 				true,
 			);

substrate/client/service/test/src/client/mod.rs

@@ -1496,7 +1496,7 @@ fn doesnt_import_blocks_that_revert_finality() {
 				state_pruning: Some(PruningMode::ArchiveAll),
 				blocks_pruning: BlocksPruning::KeepAll,
 				pruning_filters: Default::default(),
-				source: DatabaseSource::RocksDb { path: tmp.path().into(), cache_size: 1024 },
+				source: DatabaseSource::RocksDb { path: tmp.path().into(), cache_size: 1024, transaction_column_path: None },
 				metrics_registry: None,
 			},
 			u64::MAX,
@@ -1778,7 +1778,7 @@ fn returns_status_for_pruned_blocks() {
 				state_pruning: Some(PruningMode::blocks_pruning(1)),
 				blocks_pruning: BlocksPruning::KeepFinalized,
 				pruning_filters: Default::default(),
-				source: DatabaseSource::RocksDb { path: tmp.path().into(), cache_size: 1024 },
+				source: DatabaseSource::RocksDb { path: tmp.path().into(), cache_size: 1024, transaction_column_path: None },
 				metrics_registry: None,
 			},
 			u64::MAX,

substrate/client/service/test/src/lib.rs

@@ -233,7 +233,7 @@ fn node_config<E: ChainSpecExtension + Clone + 'static + Send + Sync>(
 		transaction_pool: Default::default(),
 		network: network_config,
 		keystore: KeystoreConfig::Path { path: root.join("key"), password: None },
-		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 },
+		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128, transaction_column_path: None },
 		trie_cache_maximum_size: Some(16 * 1024 * 1024),
 		warm_up_trie_cache: None,
 		state_pruning: Default::default(),