diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/records/impl/WrappedRecordBlockHashMigration.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/records/impl/WrappedRecordBlockHashMigration.java index b473fa0b7abc..64ebd701c271 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/records/impl/WrappedRecordBlockHashMigration.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/records/impl/WrappedRecordBlockHashMigration.java @@ -15,9 +15,11 @@ import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; import org.apache.logging.log4j.LogManager; @@ -30,6 +32,11 @@ * hasher state) and a recent wrapped record hashes file, validates their consistency, * computes the Merkle block hashes for the range, and writes the results back to state. * + *

If a jumpstart actually runs and successfully computes a {@link Result} above, {@link #execute} + * also truncates the on-disk wrapped record hashes file to empty (when writing to it is enabled), + * since that data has now been consumed and folded into the result. This must stay downstream of + * the read above. + * *

TODO: Delete this in the release after receiving/injecting the jumpstart historical hashes data. */ public class WrappedRecordBlockHashMigration { @@ -61,7 +68,9 @@ public record Result( "Resuming calculation of wrapped record file hashes until next attempt"; /** - * Executes the wrapped record block hash migration if enabled. + * Executes the wrapped record block hash migration if enabled, then truncates the on-disk + * wrapped record hashes file to empty if a jumpstart actually ran and consumed it (and writing + * to the file is enabled). * * @param streamMode the current stream mode * @param recordsConfig the block record stream configuration @@ -78,6 +87,20 @@ public void execute( requireNonNull(recordsConfig); requireNonNull(jumpstartConfig); + try { + runJumpstartMigration(streamMode, recordsConfig, jumpstartConfig, migrationAlreadyApplied); + } finally { + if (result != null) { + truncateHashesFileIfWritingEnabled(recordsConfig); + } + } + } + + private void runJumpstartMigration( + @NonNull final StreamMode streamMode, + @NonNull final BlockRecordStreamConfig recordsConfig, + @NonNull final BlockStreamJumpstartConfig jumpstartConfig, + final boolean migrationAlreadyApplied) { if (migrationAlreadyApplied) { if (jumpstartConfig.blockNum() < 0) { log.info("Jumpstart migration already applied (votingComplete=true) and no jumpstart config, skipping"); @@ -100,6 +123,31 @@ public void execute( } } + /** + * Truncates the on-disk wrapped record hashes file to empty when writing to it is enabled. Only + * called when a jumpstart actually ran and consumed the file's contents this execution (see + * {@link #execute}), since that is the data this truncation would otherwise discard. + */ + private void truncateHashesFileIfWritingEnabled(@NonNull final BlockRecordStreamConfig recordsConfig) { + if (!recordsConfig.writeWrappedRecordFileBlockHashesToDisk()) { + return; + } + if (isBlank(recordsConfig.wrappedRecordHashesDir())) { + return; + } + final var file = Paths.get(recordsConfig.wrappedRecordHashesDir()) + .resolve(WrappedRecordFileBlockHashesDiskWriter.DEFAULT_FILE_NAME); + if (!Files.exists(file)) { + return; + } + try (final var ignored = + Files.newByteChannel(file, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + log.info("Truncated wrapped record hashes file {} now that the jumpstart migration has run", file); + } catch (final IOException e) { + log.warn("Failed to truncate wrapped record hashes file {}", file, e); + } + } + private void executeInternal( @NonNull final BlockRecordStreamConfig recordsConfig, @NonNull final BlockStreamJumpstartConfig jumpstartConfig) diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/records/impl/WrappedRecordBlockHashMigrationTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/records/impl/WrappedRecordBlockHashMigrationTest.java index 4719e8dc92c6..1293e551c032 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/records/impl/WrappedRecordBlockHashMigrationTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/records/impl/WrappedRecordBlockHashMigrationTest.java @@ -455,6 +455,104 @@ void loadRecentHashesParsesLargeFileWithoutException() throws Exception { assertThat(result.entries().size()).isEqualTo(entryCount); } + @Test + void truncatesFileWhenWritingEnabledAfterSuccessfulMigration() throws Exception { + final List entries = new ArrayList<>(); + for (long i = 90; i <= 100; i++) { + entries.add(entry(i)); + } + final var recentHashesDir = createRecentHashesDir(entries); + final var file = recentHashesDir.resolve(WrappedRecordFileBlockHashesDiskWriter.DEFAULT_FILE_NAME); + final var config = recordsConfigWith(RECORDS, true, b -> b.withValue( + "hedera.recordStream.wrappedRecordHashesDir", recentHashesDir.toString()) + .withValue("hedera.recordStream.writeWrappedRecordFileBlockHashesToDisk", true)); + + subject.execute(StreamMode.RECORDS, config, jumpstartConfig(98, 4, 1), false); + + // The migration itself must have read the pre-truncation file contents successfully. + assertThat(subject.result()).isNotNull(); + assertThat(Files.exists(file)).isTrue(); + assertThat(Files.size(file)).isZero(); + } + + @Test + void doesNotTruncateFileWhenJumpstartNotConfigured() throws Exception { + final List entries = new ArrayList<>(); + for (long i = 90; i <= 100; i++) { + entries.add(entry(i)); + } + final var recentHashesDir = createRecentHashesDir(entries); + final var file = recentHashesDir.resolve(WrappedRecordFileBlockHashesDiskWriter.DEFAULT_FILE_NAME); + final var config = recordsConfigWith(RECORDS, true, b -> b.withValue( + "hedera.recordStream.wrappedRecordHashesDir", recentHashesDir.toString()) + .withValue("hedera.recordStream.writeWrappedRecordFileBlockHashesToDisk", true)); + + // No jumpstart config populated (blockNum defaults to -1), so the migration itself is a no-op + // and must not touch the file, even though writing is enabled. + subject.execute(StreamMode.RECORDS, config, defaultJumpstartConfig(), false); + + assertNull(subject.result()); + assertThat(Files.size(file)).isGreaterThan(0L); + } + + @Test + void doesNotTruncateFileWhenValidationFailsSoJumpstartDoesNotRun() throws Exception { + final var config = enabledRecordsConfig(createRecentHashesDir(List.of(entry(100), entry(101)))); + final var file = + tempDir.resolve("recent-hashes").resolve(WrappedRecordFileBlockHashesDiskWriter.DEFAULT_FILE_NAME); + // previousWrappedRecordBlockHash has the wrong length, so validation fails before any hashes are computed. + final var badConfig = new BlockStreamJumpstartConfig( + 100, + Bytes.wrap(new byte[32]), + 4, + 1, + List.of(Bytes.wrap(new byte[HASH_SIZE])), + Bytes.wrap(new byte[HASH_SIZE]), + Bytes.wrap(new byte[HASH_SIZE])); + + subject.execute(StreamMode.RECORDS, config, badConfig, false); + + assertNull(subject.result()); + assertThat(Files.size(file)).isGreaterThan(0L); + } + + @Test + void doesNotTruncateFileWhenWritingDisabled() throws Exception { + final List entries = new ArrayList<>(); + for (long i = 90; i <= 100; i++) { + entries.add(entry(i)); + } + final var recentHashesDir = createRecentHashesDir(entries); + final var file = recentHashesDir.resolve(WrappedRecordFileBlockHashesDiskWriter.DEFAULT_FILE_NAME); + final var config = recordsConfigWith(RECORDS, true, b -> b.withValue( + "hedera.recordStream.wrappedRecordHashesDir", recentHashesDir.toString()) + .withValue("hedera.recordStream.writeWrappedRecordFileBlockHashesToDisk", false)); + + subject.execute(StreamMode.RECORDS, config, jumpstartConfig(98, 4, 1), false); + + assertThat(subject.result()).isNotNull(); + assertThat(Files.size(file)).isGreaterThan(0L); + } + + @Test + void truncateHelperDoesNotFailWhenFileMissing() throws Exception { + final var config = recordsConfigWith(RECORDS, true, b -> b.withValue( + "hedera.recordStream.wrappedRecordHashesDir", + tempDir.resolve("missing-dir").toString()) + .withValue("hedera.recordStream.writeWrappedRecordFileBlockHashesToDisk", true)); + + final Method truncate = WrappedRecordBlockHashMigration.class.getDeclaredMethod( + "truncateHashesFileIfWritingEnabled", BlockRecordStreamConfig.class); + truncate.setAccessible(true); + assertDoesNotThrow(() -> { + try { + truncate.invoke(subject, config); + } catch (final java.lang.reflect.InvocationTargetException e) { + throw e.getCause(); + } + }); + } + private Path createRecentHashesDir(List entries) throws Exception { final var dir = tempDir.resolve("recent-hashes"); Files.createDirectories(dir); diff --git a/hedera-node/hedera-config/src/main/java/com/hedera/node/config/data/BlockRecordStreamConfig.java b/hedera-node/hedera-config/src/main/java/com/hedera/node/config/data/BlockRecordStreamConfig.java index 0841b7e6d223..da67073641aa 100644 --- a/hedera-node/hedera-config/src/main/java/com/hedera/node/config/data/BlockRecordStreamConfig.java +++ b/hedera-node/hedera-config/src/main/java/com/hedera/node/config/data/BlockRecordStreamConfig.java @@ -54,7 +54,7 @@ public record BlockRecordStreamConfig( @ConfigProperty(defaultValue = "concurrent") @NetworkProperty String streamFileProducer, - @ConfigProperty(defaultValue = "false") @NetworkProperty + @ConfigProperty(defaultValue = "true") @NetworkProperty boolean writeWrappedRecordFileBlockHashesToDisk, @ConfigProperty(defaultValue = "/opt/hgcapp/wrappedRecordHashes") @NodeProperty diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/UtilVerbs.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/UtilVerbs.java index f8d426e00014..6348e7148cbd 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/UtilVerbs.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/UtilVerbs.java @@ -959,20 +959,18 @@ public static GetWrappedRecordHashesOp getWrappedRecordHashes( } /** - * Verifies the node's jumpstart hash computation via three-way comparison: - * file entries, .rcd replay, and the node's logged hash. + * Verifies the node's jumpstart hash computation by independently replaying {@code .rcd} files + * from the jumpstart block through the freeze block and comparing against the node's logged hash. * * @param jumpstartConfig the jumpstart config properties - * @param wrappedHashes per-block entries from the wrapped record hashes file * @param nodeComputedHash the hash the node logged during migration * @param freezeBlockNum the last block the migration processed */ public static VerifyJumpstartHashOp verifyJumpstartHash( @NonNull final BlockStreamJumpstartConfig jumpstartConfig, - @NonNull final List wrappedHashes, @NonNull final String nodeComputedHash, @NonNull final String freezeBlockNum) { - return new VerifyJumpstartHashOp(jumpstartConfig, wrappedHashes, nodeComputedHash, freezeBlockNum); + return new VerifyJumpstartHashOp(jumpstartConfig, nodeComputedHash, freezeBlockNum); } /** diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/upgrade/VerifyJumpstartHashOp.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/upgrade/VerifyJumpstartHashOp.java index d50d6c77a953..8461865d6f11 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/upgrade/VerifyJumpstartHashOp.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/upgrade/VerifyJumpstartHashOp.java @@ -4,7 +4,6 @@ import static com.hedera.node.app.hapi.utils.CommonUtils.sha384DigestOrThrow; import static java.util.Objects.requireNonNull; -import com.hedera.hapi.block.internal.WrappedRecordFileBlockHashes; import com.hedera.node.app.blocks.impl.IncrementalStreamingHasher; import com.hedera.node.config.data.BlockStreamJumpstartConfig; import com.hedera.pbj.runtime.io.buffer.Bytes; @@ -12,38 +11,33 @@ import com.hedera.services.bdd.spec.utilops.UtilOp; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.ArrayList; -import java.util.Comparator; import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Assertions; /** - * Three-way verification of the jumpstart hash computation: - *

    - *
  1. Chain 1 (file entries): chains wrapped hashes file entries via - * {@link RcdFileBlockHashReplay#computeBlockRootHash}
    2. - *
  2. Chain 2 (.rcd replay): replays {@code .rcd} files via - * {@link RcdFileBlockHashReplay}
    3. - *
  3. Three-way assertions: per-block entry comparison, plus both chain hashes - * must match the node's logged hash
    4. - *
+ * Verifies the node's jumpstart hash computation by independently replaying {@code .rcd} files + * from the jumpstart block through the freeze block and comparing the final chained hash against + * the node's logged hash. + * + *

This cannot also cross-check against the wrapped record hashes file's raw entries, because + * that file is truncated to empty as soon as a jumpstart migration consumes it (see + * {@code WrappedRecordBlockHashMigration}); by the time this op runs, the entries it would need + * are already gone from disk. */ public class VerifyJumpstartHashOp extends UtilOp { private static final Logger log = LogManager.getLogger(VerifyJumpstartHashOp.class); private final BlockStreamJumpstartConfig jumpstartConfig; - private final List wrappedHashes; private final String nodeComputedHash; private final String freezeBlockNum; public VerifyJumpstartHashOp( @NonNull final BlockStreamJumpstartConfig jumpstartConfig, - @NonNull final List wrappedHashes, @NonNull final String nodeComputedHash, @NonNull final String freezeBlockNum) { this.jumpstartConfig = requireNonNull(jumpstartConfig); - this.wrappedHashes = requireNonNull(wrappedHashes); this.nodeComputedHash = requireNonNull(nodeComputedHash); this.freezeBlockNum = requireNonNull(freezeBlockNum); } @@ -51,12 +45,9 @@ public VerifyJumpstartHashOp( @Override protected boolean submitOp(@NonNull final HapiSpec spec) throws Throwable { final long freezeBlock = Long.parseLong(freezeBlockNum); - - // Create two independent hasher instances - final var hasher1 = createHasherFromConfig(jumpstartConfig); - final var hasher2 = createHasherFromConfig(jumpstartConfig); final long jumpstartBlockNum = jumpstartConfig.blockNum(); final Bytes prevHash = jumpstartConfig.previousWrappedRecordBlockHash(); + final var hasher = createHasherFromConfig(jumpstartConfig); log.info( "[VerifyJumpstartHash] Jumpstart block={}, prevHash={}, freeze block={}", @@ -64,114 +55,13 @@ protected boolean submitOp(@NonNull final HapiSpec spec) throws Throwable { prevHash, freezeBlock); - // ===== Chain 1: File entries chained via computeWrappedRecordBlockRootHash ===== - final var neededEntries = wrappedHashes.stream() - .filter(e -> e.blockNumber() > jumpstartBlockNum && e.blockNumber() <= freezeBlock) - .sorted(Comparator.comparingLong(WrappedRecordFileBlockHashes::blockNumber)) - .toList(); - - log.info( - "[VerifyJumpstartHash] Chain 1 (file): will replay {} entries, blocks ({}, {}]", - neededEntries.size(), - jumpstartBlockNum, - neededEntries.isEmpty() ? "n/a" : neededEntries.getLast().blockNumber()); - - Bytes fileChainHash = prevHash; - int index = 0; - for (final var entry : neededEntries) { - final var allPrevBlocksRootHash = Bytes.wrap(hasher1.computeRootHash()); - final var blockRootHash = - RcdFileBlockHashReplay.computeBlockRootHash(fileChainHash, allPrevBlocksRootHash, entry); - - // Log first 3 and last 3 blocks for debugging - final boolean isEdgeBlock = index < 3 || index >= neededEntries.size() - 3; - if (isEdgeBlock) { - log.info( - "[VerifyJumpstartHash] Chain 1 block {} (index {}): prevHash={}, ctHash={}, outputRoot={} ā†’ finalHash={}", - entry.blockNumber(), - index, - fileChainHash, - entry.consensusTimestampHash(), - entry.outputItemsTreeRootHash(), - blockRootHash); - } - - hasher1.addNodeByHash(blockRootHash.toByteArray()); - fileChainHash = blockRootHash; - index++; - } - - log.info("[VerifyJumpstartHash] Chain 1 (file) final hash: {}", fileChainHash); - - // ===== Chain 2: .rcd replay ===== - final var rcdResult = RcdFileBlockHashReplay.replay(spec, jumpstartBlockNum, freezeBlock, prevHash, hasher2); + final var rcdResult = RcdFileBlockHashReplay.replay(spec, jumpstartBlockNum, freezeBlock, prevHash, hasher); log.info( - "[VerifyJumpstartHash] Chain 2 (.rcd) processed {} blocks, final hash: {}", + "[VerifyJumpstartHash] .rcd replay processed {} blocks, final hash: {}", rcdResult.blocksProcessed(), rcdResult.finalChainedHash()); - // ===== Three-way assertions ===== - - // 1. Per-block: .rcd entry vs file entry (ctHash + outputRoot) - final var fileEntriesByBlock = new java.util.HashMap(); - for (final var entry : neededEntries) { - fileEntriesByBlock.put(entry.blockNumber(), entry); - } - - int mismatches = 0; - for (final var mapEntry : rcdResult.entriesByBlock().entrySet()) { - final long blockNumber = mapEntry.getKey(); - final var rcdEntry = mapEntry.getValue(); - final var fileEntry = fileEntriesByBlock.get(blockNumber); - if (fileEntry == null) { - log.warn( - "[VerifyJumpstartHash] No file entry for block {} (rcd ctHash={}, outputRoot={})", - blockNumber, - rcdEntry.consensusTimestampHash(), - rcdEntry.outputItemsTreeRootHash()); - mismatches++; - continue; - } - if (!rcdEntry.consensusTimestampHash().equals(fileEntry.consensusTimestampHash()) - || !rcdEntry.outputItemsTreeRootHash().equals(fileEntry.outputItemsTreeRootHash())) { - log.error( - "[VerifyJumpstartHash] Block {} mismatch:" - + " rcd ctHash={}, file ctHash={};" - + " rcd outputRoot={}, file outputRoot={}", - blockNumber, - rcdEntry.consensusTimestampHash(), - fileEntry.consensusTimestampHash(), - rcdEntry.outputItemsTreeRootHash(), - fileEntry.outputItemsTreeRootHash()); - mismatches++; - } - } - Assertions.assertEquals( - 0, - mismatches, - "[VerifyJumpstartHash] Found " + mismatches - + " block-level mismatches between .rcd replay and file entries"); - - // 2. File chain hash vs node logged hash - Assertions.assertEquals( - nodeComputedHash, - fileChainHash.toString(), - ("[VerifyJumpstartHash] File chain mismatch after processing %d entries (blocks %sā€“%s)." - + " jumpstart block=%d, freeze block=%d." - + " Check node logs for 'First needed record' and 'Last recent record'.") - .formatted( - neededEntries.size(), - neededEntries.isEmpty() - ? "n/a" - : neededEntries.getFirst().blockNumber(), - neededEntries.isEmpty() - ? "n/a" - : neededEntries.getLast().blockNumber(), - jumpstartBlockNum, - freezeBlock)); - - // 3. .rcd chain hash vs node logged hash Assertions.assertEquals( nodeComputedHash, rcdResult.finalChainedHash().toString(), @@ -181,8 +71,7 @@ protected boolean submitOp(@NonNull final HapiSpec spec) throws Throwable { .formatted(rcdResult.blocksProcessed(), jumpstartBlockNum, freezeBlock)); log.info( - "[VerifyJumpstartHash] Three-way verification passed: file chain={}, rcd chain={}, node logged={}", - fileChainHash, + "[VerifyJumpstartHash] Verification passed: rcd chain={}, node logged={}", rcdResult.finalChainedHash(), nodeComputedHash); diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/upgrade/VerifyLiveWrappedHashOp.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/upgrade/VerifyLiveWrappedHashOp.java index ec714be0ccef..ac3a5c1c6028 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/upgrade/VerifyLiveWrappedHashOp.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/spec/utilops/upgrade/VerifyLiveWrappedHashOp.java @@ -16,10 +16,6 @@ * Verifies live wrapped record block hashes by replaying {@code .rcd} files from * genesis through the live-hash freeze block and asserting the final chained hash * matches the node's persisted live hash. - * - *

Per-block entry verification against the wrapped hashes file is handled - * separately by {@link VerifyJumpstartHashOp} in Phase 4; this operation focuses - * solely on the end-to-end chained hash correctness. */ public class VerifyLiveWrappedHashOp extends UtilOp { diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/freeze/JumpstartFileSuite.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/freeze/JumpstartFileSuite.java index 9ea131f4f281..20d90e1471da 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/freeze/JumpstartFileSuite.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/freeze/JumpstartFileSuite.java @@ -12,7 +12,6 @@ import static com.hedera.services.bdd.spec.utilops.UtilVerbs.buildDynamicJumpstartConfig; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.doAdhoc; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.doingContextual; -import static com.hedera.services.bdd.spec.utilops.UtilVerbs.getWrappedRecordHashes; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.logIt; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.sourcing; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.verifyJumpstartHash; @@ -26,7 +25,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import com.hedera.hapi.block.internal.WrappedRecordFileBlockHashes; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.output.SingletonUpdateChange; import com.hedera.hapi.node.state.blockrecords.BlockInfo; @@ -50,7 +48,6 @@ import java.nio.file.StandardCopyOption; import java.time.Duration; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; @@ -82,7 +79,6 @@ class JumpstartFileSuite implements LifecycleTest { "blockStream.streamMode" }) final Stream executesAllCutoverPhases() { - final AtomicReference> wrappedRecordHashes = new AtomicReference<>(); final AtomicReference jumpstartConfig = new AtomicReference<>(); final AtomicReference nodeComputedHash = new AtomicReference<>(); final AtomicReference freezeBlockNum = new AtomicReference<>(); @@ -91,7 +87,6 @@ final Stream executesAllCutoverPhases() { final AtomicReference capturedBlockInfo = new AtomicReference<>(); final AtomicReference capturedRunningHashes = new AtomicReference<>(); final AtomicReference jumpstartConfig2 = new AtomicReference<>(); - final AtomicReference> wrappedRecordHashes2 = new AtomicReference<>(); final AtomicReference nodeComputedHash2 = new AtomicReference<>(); final AtomicReference freezeBlockNum2 = new AtomicReference<>(); @@ -153,16 +148,12 @@ final Stream executesAllCutoverPhases() { Duration.ofSeconds(30)) .exposingMatchGroupTo(1, freezeBlockNum) .exposingMatchGroupTo(2, nodeComputedHash), - // Independently verify the node's computed hash. The wrapped record hashes file - // may have grown since the migration ran (nodes continue writing after restart), - // so we pass the freeze block number to bound the replay to the same range the - // migration processed. - getWrappedRecordHashes(wrappedRecordHashes), - sourcing(() -> verifyJumpstartHash( - jumpstartConfig.get(), - wrappedRecordHashes.get(), - nodeComputedHash.get(), - freezeBlockNum.get())), + // Independently verify the node's computed hash by replaying .rcd files from the + // jumpstart block through the freeze block (the wrapped record hashes file itself + // is truncated to empty as soon as the jumpstart migration consumes it, so it can't + // be used for a post-hoc cross-check here). + sourcing( + () -> verifyJumpstartHash(jumpstartConfig.get(), nodeComputedHash.get(), freezeBlockNum.get())), logIt("Phase 6: Verify a SECOND jumpstart cycle re-computes a distinct, correct hash"), MixedOperations.burstOfTps(5, Duration.ofSeconds(30)), prepareFakeUpgrade(), @@ -184,12 +175,8 @@ final Stream executesAllCutoverPhases() { .matchingLast() .exposingMatchGroupTo(1, freezeBlockNum2) .exposingMatchGroupTo(2, nodeComputedHash2), - getWrappedRecordHashes(wrappedRecordHashes2), - sourcing(() -> verifyJumpstartHash( - jumpstartConfig2.get(), - wrappedRecordHashes2.get(), - nodeComputedHash2.get(), - freezeBlockNum2.get())), + sourcing(() -> + verifyJumpstartHash(jumpstartConfig2.get(), nodeComputedHash2.get(), freezeBlockNum2.get())), assertHgcaaLogContainsPattern( NodeSelector.exceptNodeIds(LATER_NODE_IDS), "Migration root hash voting finalized after node\\d+ vote, >1/3 threshold reached",