From 78cf06d021d4d79d3388946824b096a99e821716 Mon Sep 17 00:00:00 2001 From: Say Cheong Date: Fri, 29 Nov 2024 10:56:55 +0900 Subject: [PATCH 1/5] Optimized IterateIndexes() for pruned RocksDBStore --- src/Libplanet.RocksDBStore/RocksDBStore.cs | 91 ++++++++++++++++++---- 1 file changed, 75 insertions(+), 16 deletions(-) diff --git a/src/Libplanet.RocksDBStore/RocksDBStore.cs b/src/Libplanet.RocksDBStore/RocksDBStore.cs index cd0bf965539..b3ed980a490 100644 --- a/src/Libplanet.RocksDBStore/RocksDBStore.cs +++ b/src/Libplanet.RocksDBStore/RocksDBStore.cs @@ -153,6 +153,7 @@ public partial class RocksDBStore : BaseStore private readonly ReaderWriterLockSlim _rwBlockCommitLock; private readonly ReaderWriterLockSlim _rwNextStateRootHashLock; private readonly ReaderWriterLockSlim _rwEvidenceLock; + private bool _pruned = false; private bool _disposed = false; private object _chainForkDeleteLock = new object(); private LruCache>> _indexCache; @@ -283,6 +284,8 @@ public RocksDBStore( new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); _rwEvidenceLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); + _pruned = ListChainIds().Count() <= 1; + _blockDbCache = new LruCache(dbConnectionCacheSize); _blockDbCache.SetPreRemoveDataMethod(db => { @@ -310,7 +313,7 @@ public static bool MigrateChainDBFromColumnFamilies(string path) } RocksDb db = RocksDb.Open(opt, path, cfs); - if (cfs.Count() == 1 && IterateDb(db, ChainIdKeyPrefix).Any()) + if (cfs.Count() == 1 && IterateDbUnpruned(db, ChainIdKeyPrefix).Any()) { // Already migrated. db.Dispose(); @@ -419,7 +422,7 @@ void CopyIndexes(ColumnFamilyHandle cfh, long? limit) /// public override IEnumerable ListChainIds() { - foreach (var it in IterateDb(_chainDb, ChainIdKeyPrefix)) + foreach (var it in IterateDbUnpruned(_chainDb, ChainIdKeyPrefix)) { var guid = new Guid(it.Value()); if (IsDeletionMarked(guid) && HasFork(guid)) @@ -444,7 +447,7 @@ public override void DeleteChainId(Guid chainId) // FIXME: We should remove this code after adjusting .ForkTxNonces(). using var batch = new WriteBatch(); byte[] prefix = TxNonceKey(chainId); - foreach (Iterator k in IterateDb(_chainDb, prefix)) + foreach (Iterator k in IterateDbUnpruned(_chainDb, prefix)) { batch.Delete(k.Key()); } @@ -460,12 +463,12 @@ public override void DeleteChainId(Guid chainId) try { using var batch = new WriteBatch(); - foreach (Iterator it in IterateDb(_chainDb, IndexKey(chainId))) + foreach (Iterator it in IterateDbUnpruned(_chainDb, IndexKey(chainId))) { batch.Delete(it.Key()); } - foreach (Iterator it in IterateDb(_chainDb, TxNonceKey(chainId))) + foreach (Iterator it in IterateDbUnpruned(_chainDb, TxNonceKey(chainId))) { batch.Delete(it.Key()); } @@ -817,7 +820,7 @@ public override IEnumerable IterateBlockHashes() { byte[] prefix = BlockKeyPrefix; - foreach (Iterator it in IterateDb(_blockIndexDb, prefix)) + foreach (Iterator it in IterateDbUnpruned(_blockIndexDb, prefix)) { byte[] key = it.Key(); byte[] hashBytes = key.Skip(prefix.Length).ToArray(); @@ -1014,7 +1017,7 @@ public override void DeleteTxIdBlockHashIndex(TxId txId, BlockHash blockHash) public override IEnumerable IterateTxIdBlockHashIndex(TxId txId) { var prefix = TxIdBlockHashIndexTxIdKey(txId); - foreach (var it in IterateDb(_txIdBlockHashIndexDb, prefix)) + foreach (var it in IterateDbUnpruned(_txIdBlockHashIndexDb, prefix)) { yield return new BlockHash(it.Value()); } @@ -1043,7 +1046,7 @@ public override void PutTxExecution(TxExecution txExecution) => public override IEnumerable> ListTxNonces(Guid chainId) { byte[] prefix = TxNonceKey(chainId); - foreach (Iterator it in IterateDb(_chainDb, prefix)) + foreach (Iterator it in IterateDbUnpruned(_chainDb, prefix)) { byte[] addressBytes = it.Key() .Skip(prefix.Length) @@ -1296,7 +1299,8 @@ public override IEnumerable GetBlockCommitHashes() { try { - IEnumerable iterators = IterateDb(_blockCommitDb, Array.Empty()); + IEnumerable iterators = IterateDbUnpruned( + _blockCommitDb, Array.Empty()); // FIXME: Somehow key value comes with 0x76 prefix at the first index of // byte array. @@ -1396,7 +1400,7 @@ public override void DeleteNextStateRootHash(BlockHash blockHash) /// public override IEnumerable IteratePendingEvidenceIds() { - foreach (Iterator it in IterateDb(_pendingEvidenceDb, PendingEvidenceKeyPrefix)) + foreach (Iterator it in IterateDbUnpruned(_pendingEvidenceDb, PendingEvidenceKeyPrefix)) { byte[] key = it.Key(); byte[] idBytes = key.Skip(PendingEvidenceKeyPrefix.Length).ToArray(); @@ -1756,10 +1760,16 @@ private static byte[] PendingEvidenceKey(in EvidenceId evidenceId) => private static byte[] CommittedEvidenceKey(in EvidenceId evidenceId) => CommittedEvidenceKeyPrefix.Concat(evidenceId.ByteArray).ToArray(); - private static IEnumerable IterateDb(RocksDb db, byte[] prefix) + private static IEnumerable IterateDbUnpruned( + RocksDb db, + byte[] prefix, + byte[]? start = null) { using Iterator it = db.NewIterator(); - for (it.Seek(prefix); it.Valid() && it.Key().StartsWith(prefix); it.Next()) + for ( + it.Seek(start is { } s ? s : prefix); + it.Valid() && it.Key().StartsWith(prefix); + it.Next()) { yield return it; } @@ -1813,6 +1823,38 @@ private void LogUnexpectedException(string methodName, Exception e) } private IEnumerable IterateIndexes( + Guid chainId, + long offset, + long? limit, + bool includeDeleted) => _pruned + ? IterateIndexesPruned(chainId, offset, limit, includeDeleted) + : IterateIndexesUnpruned(chainId, offset, limit, includeDeleted); + + private IEnumerable IterateIndexesPruned( + Guid chainId, + long offset, + long? limit, + bool includeDeleted) + { + if (!includeDeleted && IsDeletionMarked(chainId)) + { + yield break; + } + + long count = 0; + foreach (BlockHash hash in IterateIndexesInnerPruned(chainId, offset)) + { + if (count >= limit) + { + yield break; + } + + yield return hash; + count += 1; + } + } + + private IEnumerable IterateIndexesUnpruned( Guid chainId, long offset, long? limit, @@ -1859,7 +1901,7 @@ bool includeDeleted long expectedCount = chainTipIndex - previousChainTipIndex + (GetPreviousChainInfo(cid) is null ? 1 : 0); - foreach (BlockHash hash in IterateIndexesInner(cid, expectedCount)) + foreach (BlockHash hash in IterateIndexesInnerUnpruned(cid, expectedCount)) { if (offset > 0) { @@ -1880,11 +1922,28 @@ bool includeDeleted } } - private IEnumerable IterateIndexesInner(Guid chainId, long expectedCount) + private IEnumerable IterateIndexesInnerPruned( + Guid chainId, + long offset) + { + byte[] start = Concat( + IndexKeyPrefix, + chainId.ToByteArray(), + RocksDBStoreBitConverter.GetBytes(offset)); + byte[] prefix = Concat(IndexKeyPrefix, chainId.ToByteArray()); + + foreach (Iterator it in IterateDbUnpruned(_chainDb, prefix, start)) + { + byte[] value = it.Value(); + yield return new BlockHash(value); + } + } + + private IEnumerable IterateIndexesInnerUnpruned(Guid chainId, long expectedCount) { long count = 0; byte[] prefix = Concat(IndexKeyPrefix, chainId.ToByteArray()); - foreach (Iterator it in IterateDb(_chainDb, prefix)) + foreach (Iterator it in IterateDbUnpruned(_chainDb, prefix)) { if (count >= expectedCount) { @@ -1910,7 +1969,7 @@ private void RemoveFork(Guid chainId, Guid forkedChainId) private bool HasFork(Guid chainId) { byte[] prefix = Concat(ForkedChainsKeyPrefix, chainId.ToByteArray()); - return IterateDb(_chainDb, prefix).Any(); + return IterateDbUnpruned(_chainDb, prefix).Any(); } private bool IsDeletionMarked(Guid chainId) From fb6fe3bf9f5675d9a202803a869030071590a77b Mon Sep 17 00:00:00 2001 From: Say Cheong Date: Fri, 29 Nov 2024 14:20:36 +0900 Subject: [PATCH 2/5] Further optimization --- src/Libplanet.RocksDBStore/RocksDBStore.cs | 25 +++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/Libplanet.RocksDBStore/RocksDBStore.cs b/src/Libplanet.RocksDBStore/RocksDBStore.cs index b3ed980a490..208a97af8cb 100644 --- a/src/Libplanet.RocksDBStore/RocksDBStore.cs +++ b/src/Libplanet.RocksDBStore/RocksDBStore.cs @@ -153,7 +153,6 @@ public partial class RocksDBStore : BaseStore private readonly ReaderWriterLockSlim _rwBlockCommitLock; private readonly ReaderWriterLockSlim _rwNextStateRootHashLock; private readonly ReaderWriterLockSlim _rwEvidenceLock; - private bool _pruned = false; private bool _disposed = false; private object _chainForkDeleteLock = new object(); private LruCache>> _indexCache; @@ -284,8 +283,6 @@ public RocksDBStore( new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); _rwEvidenceLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); - _pruned = ListChainIds().Count() <= 1; - _blockDbCache = new LruCache(dbConnectionCacheSize); _blockDbCache.SetPreRemoveDataMethod(db => { @@ -300,6 +297,8 @@ public RocksDBStore( }); } + private bool IsPruned => ListChainIds().Count() <= 1; + public static bool MigrateChainDBFromColumnFamilies(string path) { var opt = new DbOptions(); @@ -1826,7 +1825,7 @@ private IEnumerable IterateIndexes( Guid chainId, long offset, long? limit, - bool includeDeleted) => _pruned + bool includeDeleted) => IsPruned ? IterateIndexesPruned(chainId, offset, limit, includeDeleted) : IterateIndexesUnpruned(chainId, offset, limit, includeDeleted); @@ -1842,15 +1841,25 @@ private IEnumerable IterateIndexesPruned( } long count = 0; + long limitUpperBound = CountIndex(chainId) - offset; + long actualLimit = limit is { } l + ? Math.Min(l, limitUpperBound) + : limitUpperBound; + + if (actualLimit <= 0) + { + yield break; + } + foreach (BlockHash hash in IterateIndexesInnerPruned(chainId, offset)) { - if (count >= limit) + yield return hash; + count += 1; + + if (count >= actualLimit) { yield break; } - - yield return hash; - count += 1; } } From 3eaa3876d34eec668ee89fc52f59495685d858e6 Mon Sep 17 00:00:00 2001 From: Say Cheong Date: Fri, 29 Nov 2024 15:16:49 +0900 Subject: [PATCH 3/5] Rebase fix --- src/Libplanet.RocksDBStore/RocksDBStore.Prune.cs | 2 +- src/Libplanet.RocksDBStore/RocksDBStore.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Libplanet.RocksDBStore/RocksDBStore.Prune.cs b/src/Libplanet.RocksDBStore/RocksDBStore.Prune.cs index ae8fa2398b2..a235bd9d59c 100644 --- a/src/Libplanet.RocksDBStore/RocksDBStore.Prune.cs +++ b/src/Libplanet.RocksDBStore/RocksDBStore.Prune.cs @@ -176,7 +176,7 @@ private IEnumerable IterateIndexesForPrune(Guid chainId) private IEnumerable IterateIndexesInnerForPrune(Guid chainId) { byte[] prefix = Concat(IndexKeyPrefix, chainId.ToByteArray()); - foreach (Iterator it in IterateDb(_chainDb, prefix)) + foreach (Iterator it in IterateDbUnpruned(_chainDb, prefix)) { byte[] value = it.Value(); yield return new BlockHash(value); diff --git a/src/Libplanet.RocksDBStore/RocksDBStore.cs b/src/Libplanet.RocksDBStore/RocksDBStore.cs index 208a97af8cb..b37479f4538 100644 --- a/src/Libplanet.RocksDBStore/RocksDBStore.cs +++ b/src/Libplanet.RocksDBStore/RocksDBStore.cs @@ -666,7 +666,7 @@ BlockHash branchpoint } using var batch = new WriteBatch(); - foreach (Iterator k in IterateDb(_chainDb, IndexKey(destinationChainId))) + foreach (Iterator k in IterateDbUnpruned(_chainDb, IndexKey(destinationChainId))) { batch.Delete(k.Key()); } @@ -1147,7 +1147,7 @@ public override void ForkTxNonces(Guid sourceChainId, Guid destinationChainId) try { byte[] prefix = TxNonceKey(sourceChainId); - foreach (Iterator it in IterateDb(_chainDb, prefix)) + foreach (Iterator it in IterateDbUnpruned(_chainDb, prefix)) { exist = true; Address address = new Address(it.Key().Skip(prefix.Length).ToArray()); From 7eb63588685300809264f909ab73b34a0c1f7492 Mon Sep 17 00:00:00 2001 From: Say Cheong Date: Fri, 29 Nov 2024 15:27:40 +0900 Subject: [PATCH 4/5] Changelog --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 469b35369d1..34be114d251 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,6 +25,7 @@ To be released. ### Bug fixes - Fixed a bug in `IStore.PruneOutdatedChains()`. [[#3999]] + - Optimized `IStore.IterateIndexes()`. [[#4004]] ### Dependencies @@ -32,6 +33,7 @@ To be released. [#3997]: https://github.com/planetarium/libplanet/pull/3997 [#3999]: https://github.com/planetarium/libplanet/pull/3999 +[#4004]: https://github.com/planetarium/libplanet/pull/4004 Version 5.4.0 From d5e77b893ab88ed168d59909017b70a204041025 Mon Sep 17 00:00:00 2001 From: Say Cheong Date: Fri, 29 Nov 2024 15:45:03 +0900 Subject: [PATCH 5/5] Removed unnecessary test --- test/Libplanet.Tests/Store/StoreTest.cs | 27 ------------------------- 1 file changed, 27 deletions(-) diff --git a/test/Libplanet.Tests/Store/StoreTest.cs b/test/Libplanet.Tests/Store/StoreTest.cs index 5a25950f8b6..714b32e79b6 100644 --- a/test/Libplanet.Tests/Store/StoreTest.cs +++ b/test/Libplanet.Tests/Store/StoreTest.cs @@ -284,33 +284,6 @@ public void DeleteChainIdWithForksReverse() Assert.Empty(store.IterateIndexes(chainC)); } - [SkippableFact] - public void ForkFromChainWithDeletion() - { - IStore store = Fx.Store; - Guid chainA = Guid.NewGuid(); - Guid chainB = Guid.NewGuid(); - Guid chainC = Guid.NewGuid(); - - // We need `Block`s because `IStore` can't retrieve index(long) by block hash without - // actual block... - store.PutBlock(Fx.GenesisBlock); - store.PutBlock(Fx.Block1); - store.PutBlock(Fx.Block2); - store.PutBlock(Fx.Block3); - - store.AppendIndex(chainA, Fx.GenesisBlock.Hash); - store.AppendIndex(chainA, Fx.Block1.Hash); - store.ForkBlockIndexes(chainA, chainB, Fx.Block1.Hash); - store.DeleteChainId(chainA); - - store.ForkBlockIndexes(chainB, chainC, Fx.Block1.Hash); - Assert.Equal( - Fx.Block1.Hash, - store.IndexBlockHash(chainC, Fx.Block1.Index) - ); - } - [SkippableFact] public void CanonicalChainId() {