diff --git a/AGENTS.md b/AGENTS.md index 6cfb76e489..0b1f28559a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,8 +30,3 @@ - Follow existing test patterns in similar files - Type annotations for public methods - Prefer immutable data structures and functional patterns - -## Development Restrictions -- **Code Changes**: Only modify code in `src/test/` folders -- **Production Code**: Do not touch production code in `src/main/` directories -- **Test Focus**: All development work should be test-related only \ No newline at end of file diff --git a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/extension/ExtensionCandidate.scala b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/extension/ExtensionCandidate.scala index 61513e3360..370f921d7b 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/extension/ExtensionCandidate.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/extension/ExtensionCandidate.scala @@ -6,7 +6,6 @@ import scorex.crypto.authds.LeafData import scorex.crypto.authds.merkle.{BatchMerkleProof, Leaf, MerkleProof, MerkleTree} import scorex.crypto.hash.Digest32 import scorex.util.ModifierId -import scala.annotation.nowarn import scala.collection.mutable /** * Extension block section with header id not provided @@ -38,20 +37,18 @@ class ExtensionCandidate(val fields: Seq[(Array[Byte], Array[Byte])]) { .flatMap(kv => merkleTree.proofByElement(Leaf[Digest32](LeafData @@ kv)(Algos.hash))) /** - * Constructs BatchMerkleProof for a list of interlinks - * Note - only accounts for interlink vector fields in the extension + * Constructs a BatchMerkleProof for the requested extension fields * * @param keys - array of 2-byte keys * @return BatchMerkleProof or None if keys not found */ - @nowarn def batchProofFor(keys: Array[Byte]*): Option[BatchMerkleProof[Digest32]] = { val indices = keys.flatMap(key => fields.find(_._1 sameElements key) .map(Extension.kvToLeaf) .map(kv => Leaf[Digest32](LeafData @@ kv)(Algos.hash).hash) - .flatMap(leafData => interlinksMerkleTree.elementsHashIndex.get( + .flatMap(leafData => merkleTree.elementsHashIndex.get( new mutable.WrappedArray.ofByte(leafData)))) - if (indices.isEmpty) None else interlinksMerkleTree.proofByIndices(indices)(Algos.hash) + if (indices.isEmpty) None else merkleTree.proofByIndices(indices)(Algos.hash) } } diff --git a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowAlgos.scala b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowAlgos.scala index a441cfe9ff..abcee591a2 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowAlgos.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowAlgos.scala @@ -67,7 +67,9 @@ class NipopowAlgos(val chainSettings: ChainSettings) { */ def maxLevelOf(header: Header): Int = if (!header.isGenesis) { - val requiredTarget = org.ergoplatform.mining.q / DifficultySerializer.decodeCompactBits(header.nBits) + val decodedDifficulty = DifficultySerializer.decodeCompactBits(header.nBits) + require(decodedDifficulty > 0, "Decoded difficulty target must be positive") + val requiredTarget = org.ergoplatform.mining.q / decodedDifficulty val realTarget = powScheme.powHit(header).doubleValue val level = log2(requiredTarget.doubleValue) - log2(realTarget.doubleValue) level.toInt @@ -75,6 +77,8 @@ class NipopowAlgos(val chainSettings: ChainSettings) { Int.MaxValue } + def hasValidPow(header: Header): Boolean = powScheme.validate(header).isSuccess + /** * Computes best score of a given chain. * The score value depends on number of ยต-superblocks in the given chain. @@ -96,6 +100,8 @@ class NipopowAlgos(val chainSettings: ChainSettings) { * end function */ def bestArg(chain: Seq[Header])(m: Int): Int = { + PoPowParams.requireValidM(m) + @scala.annotation.tailrec def loop(level: Int, acc: Seq[(Int, Int)] = Seq.empty): Seq[(Int, Int)] = if (level == 0) { @@ -130,7 +136,7 @@ class NipopowAlgos(val chainSettings: ChainSettings) { val k = params.k val m = params.m - require(params.k >= 1, s"$k < 1") + PoPowParams.requireValid(m, k) require(chain.lengthCompare(k + m) >= 0, s"Can not prove chain of size < ${k + m}") require(chain.head.header.isGenesis, "Can not prove non-anchored chain") diff --git a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProof.scala b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProof.scala index c963877ada..2a8838bf2e 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProof.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProof.scala @@ -51,7 +51,9 @@ case class NipopowProof(popowAlgos: NipopowAlgos, */ def isBetterThan(that: NipopowProof): Boolean = { try { - if (this.isValid && that.isValid) { + if (this.m != that.m || this.k != that.k) { + false + } else if (this.isValid && that.isValid) { popowAlgos.lowestCommonAncestor(headersChain, that.headersChain) .map(h => headersChain.filter(_.height > h.height) -> that.headersChain.filter(_.height > h.height)) .exists({ case (thisDivergingChain, thatDivergingChain) => @@ -72,7 +74,20 @@ case class NipopowProof(popowAlgos: NipopowAlgos, * @return true if the proof is valid */ lazy val isValid: Boolean = { - this.hasValidConnections && this.hasValidHeights && this.hasValidProofs && this.hasValidDifficultyHeaders + this.hasValidParams && + this.hasValidConnections && + this.hasValidHeights && + this.hasValidProofs && + this.hasValidDifficultyHeaders && + this.hasValidPow + } + + /** + * Checks proof parameters and the exact suffix cardinality before any + * parameter-dependent scoring or validation work. + */ + lazy val hasValidParams: Boolean = { + PoPowParams.areValid(m, k) && suffixTail.lengthCompare(k - 1) == 0 } /** @@ -155,6 +170,8 @@ case class NipopowProof(popowAlgos: NipopowAlgos, suffixHead.checkInterlinksProof() } + lazy val hasValidPow: Boolean = headersChain.forall(popowAlgos.hasValidPow) + } object NipopowProof { @@ -185,9 +202,27 @@ object NipopowProof { } +object NipopowProofSerializer { + private val MaxProofElements = PoPowParams.MaxProofElements + + private def requireWithinLimit(value: Int, limit: Int, what: String): Unit = + require(value <= limit, s"$what $value exceeds sanity limit $limit") + + private def readLengthPrefixed[T](r: Reader, + limit: Int, + what: String) + (parse: Array[Byte] => T): T = { + val declaredLength = r.getUInt().toIntExact + requireWithinLimit(declaredLength, limit, s"$what length") + parse(r.getBytes(declaredLength)) + } +} + class NipopowProofSerializer(poPowAlgos: NipopowAlgos) extends ErgoSerializer[NipopowProof] { + import NipopowProofSerializer._ override def serialize(obj: NipopowProof, w: Writer): Unit = { + require(obj.hasValidParams, "Invalid NiPoPoW proof parameters or suffix length") w.putUInt(obj.m.toLong) w.putUInt(obj.k.toLong) w.putUInt(obj.prefix.size.toLong) @@ -211,19 +246,27 @@ class NipopowProofSerializer(poPowAlgos: NipopowAlgos) extends ErgoSerializer[Ni override def parse(r: Reader): NipopowProof = { val m = r.getUInt().toIntExact val k = r.getUInt().toIntExact + PoPowParams.requireValid(m, k) val prefixSize = r.getUInt().toIntExact + requireWithinLimit(prefixSize, MaxProofElements, "prefix count") val prefix = (0 until prefixSize).map { _ => - val size = r.getUInt().toIntExact - PoPowHeaderSerializer.parseBytes(r.getBytes(size)) + readLengthPrefixed(r, PoPowHeaderSerializer.MaxSerializedBytes, "prefix element")( + PoPowHeaderSerializer.parseBytes) } - val suffixHeadSize = r.getUInt().toIntExact - val suffixHead = PoPowHeaderSerializer.parseBytes(r.getBytes(suffixHeadSize)) + val suffixHead = readLengthPrefixed(r, PoPowHeaderSerializer.MaxSerializedBytes, "suffix head")( + PoPowHeaderSerializer.parseBytes) val suffixSize = r.getUInt().toIntExact + requireWithinLimit(suffixSize, MaxProofElements, "suffix count") + require(suffixSize == k - 1, + s"NiPoPoW suffix length ${suffixSize + 1} does not match k parameter $k") val suffixTail = (0 until suffixSize).map { _ => - val size = r.getUInt().toIntExact - HeaderSerializer.parseBytes(r.getBytes(size)) + readLengthPrefixed(r, PoPowHeaderSerializer.MaxHeaderBytes, "suffix tail")( + HeaderSerializer.parseBytes) } - val continuous = if (r.getByte() == 1) true else false + val continuousByte = r.getByte() + require(continuousByte == 0 || continuousByte == 1, + s"invalid NiPoPoW continuous mode byte ${continuousByte & 0xff}") + val continuous = continuousByte == 1 NipopowProof(poPowAlgos, m, k, prefix, suffixHead, suffixTail, continuous) } diff --git a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowHeader.scala b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowHeader.scala index 80bcccc649..8bd1598cca 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowHeader.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowHeader.scala @@ -1,25 +1,25 @@ package org.ergoplatform.modifiers.history.popow import cats.syntax.either._ -import sigmastate.utils.Helpers._ import cats.Traverse import cats.implicits.{catsStdInstancesForEither, catsStdInstancesForList} import io.circe.{Decoder, Encoder, Json} import org.ergoplatform.core.BytesSerializable import org.ergoplatform.modifiers.ErgoFullBlock -import org.ergoplatform.modifiers.history.extension.Extension.merkleTree +import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} import org.ergoplatform.settings.Algos import org.ergoplatform.settings.Algos.HF import org.ergoplatform.serialization.ErgoSerializer -import scorex.crypto.authds.Side -import scorex.crypto.authds.merkle.BatchMerkleProof +import scorex.crypto.authds.{LeafData, Side} +import scorex.crypto.authds.merkle.{BatchMerkleProof, Leaf} import scorex.crypto.authds.merkle.serialization.BatchMerkleProofSerializer import scorex.crypto.hash.Digest32 import scorex.util.Extensions._ import scorex.util.serialization.{Reader, Writer} import scorex.util.{ModifierId, bytesToId, idToBytes} +import java.nio.ByteBuffer import scala.util.Try /** @@ -42,7 +42,16 @@ case class PoPowHeader(header: Header, def height: Int = header.height - def checkInterlinksProof(): Boolean = PoPowHeader.checkInterlinksProof(interlinks, interlinksProof) + def checkInterlinksProof(): Boolean = { + val proofIsEmpty = interlinksProof.indices.isEmpty && interlinksProof.proofs.isEmpty + if (header.isGenesis) { + interlinks.isEmpty && proofIsEmpty + } else if (!PoPowHeader.hasCanonicalInterlinkRuns(interlinks)) { + false + } else { + PoPowHeader.checkInterlinksProof(interlinks, interlinksProof, header.extensionRoot) + } + } } object PoPowHeader { @@ -51,16 +60,61 @@ object PoPowHeader { implicit val hf: HF = Algos.hash + private[popow] def hasCanonicalInterlinkRuns(interlinks: Seq[ModifierId]): Boolean = { + interlinks.headOption.exists { first => + var current = first + var runLength = 1 + var closedRuns = Set.empty[ModifierId] + + interlinks.iterator.zipWithIndex.drop(1).forall { case (interlink, position) => + if (interlink == current) { + if (runLength == 255) { + false + } else { + runLength += 1 + true + } + } else if (position > 255) { + false + } else { + closedRuns += current + if (closedRuns.contains(interlink)) { + false + } else { + current = interlink + runLength = 1 + true + } + } + } + } + } + /** - * Validates interlinks merkle root against provided proof + * Validates the exact packed interlink leaves against the full extension root */ - def checkInterlinksProof(interlinks: Seq[ModifierId], proof: BatchMerkleProof[Digest32]): Boolean = { - if (interlinks.isEmpty && proof.indices.isEmpty && proof.proofs.isEmpty) { - true + def checkInterlinksProof(interlinks: Seq[ModifierId], + proof: BatchMerkleProof[Digest32], + extensionRoot: Digest32): Boolean = { + if (!PoPowHeaderSerializer.hasValidMerkleProofStructure( + proof.indices.map(_._1), + proof.proofs.size + )) { + false } else { - val fields = NipopowAlgos.packInterlinks(interlinks) - val tree = merkleTree(fields) - proof.valid(tree.rootHash) + Try { + val expectedLeafHashes = NipopowAlgos.packInterlinks(interlinks) + .map(Extension.kvToLeaf) + .map(kv => Leaf[Digest32](LeafData @@ kv)(Algos.hash).hash) + val provenLeafHashes = proof.indices.map(_._2) + + interlinks.nonEmpty && + expectedLeafHashes.size == provenLeafHashes.size && + expectedLeafHashes.zip(provenLeafHashes).forall { case (expected, proven) => + expected sameElements proven + } && + proof.valid(extensionRoot) + }.getOrElse(false) } } @@ -142,9 +196,87 @@ object PoPowHeader { object PoPowHeaderSerializer extends ErgoSerializer[PoPowHeader] { import org.ergoplatform.sdk.wallet.Constants.ModifierIdLength + // Generous wire sanity limits shared with the sigma-rust NiPoPoW parser. + private[ergoplatform] final val MaxHeaderBytes = 10000 + private[ergoplatform] final val MaxInterlinks = 10000 + private[ergoplatform] final val MaxMerkleProofBytes = 1000000 + private[ergoplatform] final val MaxSerializedBytes = + MaxHeaderBytes + MaxInterlinks * ModifierIdLength + MaxMerkleProofBytes + 64 + + private val MerkleProofCountBytes = 8 + private val MerkleIndexBytes = 36 + private val MerkleProofNodeBytes = 33 + private[ergoplatform] final val MaxMerkleProofDepth = + Extension.FieldKeySize * java.lang.Byte.SIZE + private val MaxMerkleLeafIndex = (1 << MaxMerkleProofDepth) - 1 + implicit val hf: HF = Algos.hash val merkleProofSerializer = new BatchMerkleProofSerializer[Digest32, HF] + private def requireWithinLimit(value: Int, limit: Int, what: String): Unit = + require(value <= limit, s"$what $value exceeds sanity limit $limit") + + private[ergoplatform] def hasValidMerkleProofStructure(indices: Seq[Int], + proofCount: Int): Boolean = { + if (indices.isEmpty) { + proofCount == 0 + } else if (proofCount < 0 || + indices.exists(index => index < 0 || index > MaxMerkleLeafIndex) || + indices.distinct.size != indices.size) { + false + } else { + var current = indices.sorted.toVector + var remainingProofs = proofCount + var depth = 0 + var valid = true + + while (valid && !(current.size == 1 && current.head == 0 && remainingProofs == 0) && + depth < MaxMerkleProofDepth) { + val currentIndices = current.toSet + val missingSiblings = current.count(index => !currentIndices.contains(index ^ 1)) + if (missingSiblings > remainingProofs) { + valid = false + } else { + remainingProofs -= missingSiblings + current = current.map(_ / 2).distinct + depth += 1 + } + } + + valid && current.size == 1 && current.head == 0 && remainingProofs == 0 + } + } + + private def validateMerkleProofPayload(bytes: Array[Byte]): Unit = { + require(bytes.length >= MerkleProofCountBytes, + s"Merkle proof counts require at least $MerkleProofCountBytes bytes") + // BatchMerkleProofSerializer stores both counts as fixed-width big-endian ints. + val counts = ByteBuffer.wrap(bytes) + val indexCount = counts.getInt + val proofCount = counts.getInt + require(indexCount >= 0 && proofCount >= 0, + "Merkle proof counts must be non-negative") + + val indexBytes = Math.multiplyExact(indexCount.toLong, MerkleIndexBytes.toLong) + val proofBytes = Math.multiplyExact(proofCount.toLong, MerkleProofNodeBytes.toLong) + val requiredBytes = Math.addExact( + MerkleProofCountBytes.toLong, + Math.addExact(indexBytes, proofBytes) + ) + require(requiredBytes == bytes.length.toLong, + s"Merkle proof counts require $requiredBytes bytes, payload has ${bytes.length}") + + val indices = Vector.newBuilder[Int] + var index = 0 + while (index < indexCount) { + indices += counts.getInt + counts.position(counts.position() + MerkleIndexBytes - Integer.BYTES) + index += 1 + } + require(hasValidMerkleProofStructure(indices.result(), proofCount), + "Merkle proof structure exceeds the extension key space") + } + override def serialize(obj: PoPowHeader, w: Writer): Unit = { val headerBytes = obj.header.bytes w.putUInt(headerBytes.length.toLong) @@ -157,12 +289,17 @@ object PoPowHeaderSerializer extends ErgoSerializer[PoPowHeader] { } override def parse(r: Reader): PoPowHeader = { - val headerSize = r.getUInt().toIntExact - val header = HeaderSerializer.parseBytes(r.getBytes(headerSize)) + val headerLength = r.getUInt().toIntExact + requireWithinLimit(headerLength, MaxHeaderBytes, "header length") + val header = HeaderSerializer.parseBytes(r.getBytes(headerLength)) val linksQty = r.getUInt().toIntExact + requireWithinLimit(linksQty, MaxInterlinks, "interlink count") val interlinks = (0 until linksQty).map(_ => bytesToId(r.getBytes(ModifierIdLength))) - val interlinksProofSize = r.getUInt().toIntExact - val interlinksProof = merkleProofSerializer.deserialize(r.getBytes(interlinksProofSize)).get + val interlinksProofLength = r.getUInt().toIntExact + requireWithinLimit(interlinksProofLength, MaxMerkleProofBytes, "Merkle proof length") + val proofBytes = r.getBytes(interlinksProofLength) + validateMerkleProofPayload(proofBytes) + val interlinksProof = merkleProofSerializer.deserialize(proofBytes).get PoPowHeader(header, interlinks, interlinksProof) } diff --git a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowParams.scala b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowParams.scala index 93aacc7339..cc7a79379a 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowParams.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowParams.scala @@ -1,5 +1,7 @@ package org.ergoplatform.modifiers.history.popow +import scala.util.Try + /** * NiPoPoW proof params from the KMZ17 paper * @@ -12,5 +14,36 @@ package org.ergoplatform.modifiers.history.popow * to the block header) * */ -case class PoPowParams(m: Int, k: Int, continuous: Boolean) +final class PoPowParams private (val m: Int, val k: Int, val continuous: Boolean, val minChainLength: Int) + +object PoPowParams { + final val MaxProofElements: Int = 20000 + + def isValidM(m: Int): Boolean = m >= 1 && m <= MaxProofElements + + def isValidK(k: Int): Boolean = k >= 1 && k <= MaxProofElements + + def isValid(m: Int, k: Int): Boolean = + isValidM(m) && isValidK(k) && m.toLong + k.toLong <= Int.MaxValue + + def areValid(m: Int, k: Int): Boolean = isValid(m, k) + + def requireValidM(m: Int): Unit = + require(isValidM(m), s"m parameter $m must be in 1..=$MaxProofElements") + + def requireValidK(k: Int): Unit = + require(isValidK(k), s"k parameter $k must be in 1..=$MaxProofElements") + + def requireValid(m: Int, k: Int): Unit = { + requireValidM(m) + requireValidK(k) + require(m.toLong + k.toLong <= Int.MaxValue, + s"NiPoPoW parameter sum exceeds Int range: m=$m, k=$k") + } + + def apply(m: Int, k: Int, continuous: Boolean): Try[PoPowParams] = Try { + requireValid(m, k) + new PoPowParams(m, k, continuous, m + k) + } +} diff --git a/ergo-core/src/test/resources/nipopow-full-root-mixed-nipopow-proof.json b/ergo-core/src/test/resources/nipopow-full-root-mixed-nipopow-proof.json new file mode 100644 index 0000000000..1a2f585ece --- /dev/null +++ b/ergo-core/src/test/resources/nipopow-full-root-mixed-nipopow-proof.json @@ -0,0 +1,13 @@ +{ + "format": "scorex-nipopow-proof-with-jvm-mode-v1", + "m": 1, + "k": 2, + "prefix_count": 1, + "suffix_count": 2, + "suffix_tail_count": 1, + "bytes_hex": "010201e601da01020000000000000000000000000000000000000000000000000000000000000000d882aaf42e0a95eb95fcce5c3705adf758e591532f733efe790ac3c404730c3963eaa9aff76a1de3d71c81e4b2d92e8d97ae572a8e9ab9e66599ed0912dd2f8b8ad868627ea4f7de6e2a2fe3f98fafe57f914e0f2ef3331c006def36c697f92713f884ebfd8e2f0e5751c026e543b2e8ab2eb06099daa1d1e5df47778f7787faab45cdf12fe3a8070239b8010400000002b3a06d6eaa8671431ba1db4dd427a77f75a5c2acbd71bfb725d38adc2b55f6695939ecfee6b0d7f400080000000000000000b103da01028022567408919e0c11029c17f56ee3f1c9567eee06dbf265ab68129d6fa2e6e0d882aaf42e0a95eb95fcce5c3705adf758e591532f733efe790ac3c404730c3963eaa9aff76a1de3d71c81e4b2d92e8d97ae572a8e9ab9e66599ed0912dd2f8b8ad868627ea4f7de6e2a2fe3f98fafe57f914e0f2ef3331c006def36c697f92713f884ebfd8e2f65fbd9d46a9392fbb9dfd6a7119516353e32755e9f4321c3810787b59cbaeccc070239b8020400000002b3a06d6eaa8671431ba1db4dd427a77f75a5c2acbd71bfb725d38adc2b55f6695939ecfee6b0d7f402111111111111111111111111111111111111111111111111111111111111111122222222222222222222222222222222222222222222222222222222222222229201000000020000000200000001fc95d4accfa4598b6151a1f9837fe4f85b028154176351a076b954abd74ae45300000002413bd0b194cedf41d6ad4ca6b0236b59ff2e070fbbfc499fda2b6f55aa27dc87580c42b47e2deed9aa364fadb6192e03715e87108fd3c96b74c21fa889baa5200000000000000000000000000000000000000000000000000000000000000000000101da01029e3967f6e21ebe3558fa02626efe43df348cb5a7b2eacd4c2853f1df9e34c20dd882aaf42e0a95eb95fcce5c3705adf758e591532f733efe790ac3c404730c3963eaa9aff76a1de3d71c81e4b2d92e8d97ae572a8e9ab9e66599ed0912dd2f8b8ad868627ea4f7de6e2a2fe3f98fafe57f914e0f2ef3331c006def36c697f92713f884ebfd8e2f65fbd9d46a9392fbb9dfd6a7119516353e32755e9f4321c3810787b59cbaeccc070239b8030400000002b3a06d6eaa8671431ba1db4dd427a77f75a5c2acbd71bfb725d38adc2b55f6695939ecfee6b0d7f400", + "rust_core_length": 891, + "terminal_continuous_byte": 0, + "sha256": "7dc6238407b20e62ef5188b331ac789836a9f43d35603e5cb21138ae6b57c0fb", + "extension_root": "65fbd9d46a9392fbb9dfd6a7119516353e32755e9f4321c3810787b59cbaeccc" +} diff --git a/ergo-core/src/test/resources/nipopow-full-root-mixed-popow-header.json b/ergo-core/src/test/resources/nipopow-full-root-mixed-popow-header.json new file mode 100644 index 0000000000..5aab2d98b1 --- /dev/null +++ b/ergo-core/src/test/resources/nipopow-full-root-mixed-popow-header.json @@ -0,0 +1,26 @@ +{ + "format": "scorex-popow-header-v1", + "label": "synthetic-mixed-full-root", + "bytes_hex": "dc01026481752bace5fa5acba5d5ef7124d48826664742d46c974c98a2d60ace229a34d882aaf42e0a95eb95fcce5c3705adf758e591532f733efe790ac3c404730c3963eaa9aff76a1de3d71c81e4b2d92e8d97ae572a8e9ab9e66599ed0912dd2f8b8ad868627ea4f7de6e2a2fe3f98fafe57f914e0f2ef3331c006def36c697f92713f884ebfd8e2f65fbd9d46a9392fbb9dfd6a7119516353e32755e9f4321c3810787b59cbaeccc070239b8c2e51c0400000002b3a06d6eaa8671431ba1db4dd427a77f75a5c2acbd71bfb725d38adc2b55f6695939ecfee6b0d7f402111111111111111111111111111111111111111111111111111111111111111122222222222222222222222222222222222222222222222222222222222222229201000000020000000200000001fc95d4accfa4598b6151a1f9837fe4f85b028154176351a076b954abd74ae45300000002413bd0b194cedf41d6ad4ca6b0236b59ff2e070fbbfc499fda2b6f55aa27dc87580c42b47e2deed9aa364fadb6192e03715e87108fd3c96b74c21fa889baa52000000000000000000000000000000000000000000000000000000000000000000001", + "length": 435, + "sha256": "832426d3beece84df9707247919b3a62444c188ae63951273f198952e8ceb068", + "extension_root": "65fbd9d46a9392fbb9dfd6a7119516353e32755e9f4321c3810787b59cbaeccc", + "extension_fields": [ + { + "key": "0000", + "value": "2a" + }, + { + "key": "0100", + "value": "011111111111111111111111111111111111111111111111111111111111111111" + }, + { + "key": "0101", + "value": "012222222222222222222222222222222222222222222222222222222222222222" + } + ], + "interlinks": [ + "1111111111111111111111111111111111111111111111111111111111111111", + "2222222222222222222222222222222222222222222222222222222222222222" + ] +} diff --git a/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/ExtensionCandidateTest.scala b/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/ExtensionCandidateTest.scala index cff39a1c2e..e4ca3ec592 100644 --- a/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/ExtensionCandidateTest.scala +++ b/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/ExtensionCandidateTest.scala @@ -4,6 +4,7 @@ import org.ergoplatform.modifiers.history.extension.ExtensionCandidate import org.ergoplatform.modifiers.history.popow.NipopowAlgos import org.ergoplatform.utils.ErgoCorePropertyTest import org.scalacheck.Gen +import scorex.util.bytesToId class ExtensionCandidateTest extends ErgoCorePropertyTest { import org.ergoplatform.utils.generators.CoreObjectGenerators.modifierIdGen @@ -40,6 +41,22 @@ class ExtensionCandidateTest extends ErgoCorePropertyTest { } } + property("batchProofFor should bind interlinks to the complete mixed extension root") { + val interlinks = Seq( + bytesToId(Array.fill(32)(1.toByte)), + bytesToId(Array.fill(32)(2.toByte)) + ) + val interlinkFields = NipopowAlgos.packInterlinks(interlinks) + val nonInterlinkField = Array[Byte](2, 0) -> Array[Byte](1) + val ext = ExtensionCandidate(interlinkFields :+ nonInterlinkField) + + val proof = ext.batchProofFor(interlinkFields.map(_._1.clone).toArray: _*) + + proof shouldBe defined + proof.get.valid(ext.digest) shouldBe true + proof.get.valid(ext.interlinksDigest) shouldBe false + } + property("batchProofFor should return None for a empty fields") { val fields: Seq[KV] = Seq.empty val ext = ExtensionCandidate(fields) diff --git a/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/NipopowProofInteropSpec.scala b/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/NipopowProofInteropSpec.scala new file mode 100644 index 0000000000..48c1833f75 --- /dev/null +++ b/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/NipopowProofInteropSpec.scala @@ -0,0 +1,251 @@ +package org.ergoplatform.modifiers.history + +import io.circe.Decoder +import io.circe.HCursor +import org.ergoplatform.modifiers.history.header.Header +import org.ergoplatform.modifiers.history.popow.NipopowAlgos +import org.ergoplatform.modifiers.history.popow.NipopowProof +import org.ergoplatform.modifiers.history.popow.NipopowProofSerializer +import org.ergoplatform.modifiers.history.popow.PoPowHeader +import org.ergoplatform.modifiers.history.popow.PoPowHeaderSerializer +import org.ergoplatform.utils.ErgoCorePropertyTest +import scorex.util.ModifierId +import scorex.util.bytesToId +import scorex.util.encode.Base16 + +import java.security.MessageDigest +import scala.io.Source +import scala.util.Try + +class NipopowProofInteropSpec extends ErgoCorePropertyTest { + import org.ergoplatform.utils.ErgoCoreTestConstants._ + + private case class ByteRange(start: Int, endExclusive: Int) + + private val FixtureResource = "nipopow-full-root-mixed-nipopow-proof.json" + private val HeaderFixtureResource = "nipopow-full-root-mixed-popow-header.json" + private val FixtureFormat = "scorex-nipopow-proof-with-jvm-mode-v1" + + private val serializer = new NipopowProofSerializer(nipopowAlgos) + + private def resourceCursor(resource: String): HCursor = { + val stream = Option(getClass.getClassLoader.getResourceAsStream(resource)) + .getOrElse(throw new IllegalArgumentException(s"Missing resource: $resource")) + val source = Source.fromInputStream(stream, "UTF-8") + val text = try source.mkString finally source.close() + io.circe.parser.parse(text).fold(error => throw error, _.hcursor) + } + + private lazy val fixture = resourceCursor(FixtureResource) + + private def fixtureValue[A: Decoder](field: String): A = + fixture.get[A](field).fold(error => throw error, identity) + + private lazy val fixtureBytes: Array[Byte] = + Base16.decode(fixtureValue[String]("bytes_hex")).get + + private lazy val rustCoreLength: Int = fixtureValue[Int]("rust_core_length") + + private lazy val terminalMode: Int = fixtureValue[Int]("terminal_continuous_byte") + + private def deterministicId(value: Byte): ModifierId = bytesToId(Array.fill(32)(value)) + + private def deterministicProof(): NipopowProof = { + val mixedFixture = resourceCursor(HeaderFixtureResource) + val mixedBytes = Base16.decode( + mixedFixture.get[String]("bytes_hex").fold(error => throw error, identity)).get + val mixedHeader = PoPowHeaderSerializer.parseBytes(mixedBytes) + + val emptyExtension = nipopowAlgos.interlinksToExtension(Seq.empty) + val emptyProof = NipopowAlgos.proofForInterlinkVector(emptyExtension).get + val genesisHeader: Header = mixedHeader.header.copy( + parentId = deterministicId(0), + height = 1, + extensionRoot = emptyExtension.digest, + sizeOpt = None + ) + val genesis = PoPowHeader(genesisHeader, Seq.empty, emptyProof) + val suffixHead = mixedHeader.copy(header = mixedHeader.header.copy( + parentId = genesis.id, + height = 2, + sizeOpt = None + )) + val suffixTail = mixedHeader.header.copy( + parentId = suffixHead.id, + height = 3, + sizeOpt = None + ) + + NipopowProof( + nipopowAlgos, + m = 1, + k = 2, + prefix = Seq(genesis), + suffixHead = suffixHead, + suffixTail = Seq(suffixTail), + continuous = false + ) + } + + /** Validate the fixture envelope without changing generic parser semantics. */ + private def parseFixtureEnvelope(bytes: Array[Byte], + coreLength: Int, + expectedTerminalMode: Int): Try[NipopowProof] = Try { + require(expectedTerminalMode == 0 || expectedTerminalMode == 1, + s"invalid JVM terminal mode $expectedTerminalMode") + require(bytes.length == coreLength + 1, + s"expected one terminal byte after $coreLength core bytes, got ${bytes.length}") + val actualTerminalMode = bytes(coreLength) & 0xff + require(actualTerminalMode == expectedTerminalMode, + s"JVM terminal mode $actualTerminalMode does not match $expectedTerminalMode") + + val parsed = serializer.parseBytes(bytes) + require(parsed.continuous == (expectedTerminalMode == 1), + "parsed JVM terminal mode differs from the fixture envelope") + require(serializer.toBytes(parsed).sameElements(bytes), + "JVM proof does not reserialize to the exact fixture envelope") + require(parsed.isValid, "parsed NiPoPoW proof is invalid") + parsed + } + + private def readFixtureVlq(bytes: Array[Byte], initialOffset: Int): (Long, Int) = { + var value = 0L + var offset = initialOffset + var shift = 0 + while (shift < 35) { + require(offset < bytes.length, "truncated fixture VLQ") + val next = bytes(offset) & 0xff + offset += 1 + value |= (next & 0x7f).toLong << shift + if ((next & 0x80) == 0) return value -> offset + shift += 7 + } + throw new IllegalArgumentException("fixture VLQ exceeds u32") + } + + private def suffixHeadRange(bytes: Array[Byte]): ByteRange = { + var offset = 0 + offset = readFixtureVlq(bytes, offset)._2 + offset = readFixtureVlq(bytes, offset)._2 + val (prefixCount, afterPrefixCount) = readFixtureVlq(bytes, offset) + offset = afterPrefixCount + (0L until prefixCount).foreach { _ => + val (payloadLength, afterPayloadLength) = readFixtureVlq(bytes, offset) + offset = afterPayloadLength + require(payloadLength <= bytes.length - offset, "prefix payload exceeds fixture bytes") + offset += payloadLength.toInt + } + val (suffixHeadLength, suffixHeadStart) = readFixtureVlq(bytes, offset) + require(suffixHeadLength <= bytes.length - suffixHeadStart, + "suffix-head payload exceeds fixture bytes") + ByteRange(suffixHeadStart, suffixHeadStart + suffixHeadLength.toInt) + } + + private def singleSubsliceOffset(bytes: Array[Byte], + range: ByteRange, + needle: Array[Byte]): Int = { + require(needle.nonEmpty, "fixture mutation target cannot be empty") + require(needle.length <= range.endExclusive - range.start, + "fixture mutation target exceeds its search range") + val offsets = (range.start to range.endExclusive - needle.length).filter { offset => + bytes.slice(offset, offset + needle.length).sameElements(needle) + } + require(offsets.length == 1, "fixture mutation target must be unique") + offsets.head + } + + property("the JVM producer reproduces the complete frozen NiPoPoW fixture") { + val produced = serializer.toBytes(deterministicProof()) + + produced shouldBe fixtureBytes + Base16.encode(MessageDigest.getInstance("SHA-256").digest(produced)) shouldBe + fixtureValue[String]("sha256") + } + + property("the complete fixture round-trips at the explicit Rust core boundary") { + fixtureValue[String]("format") shouldBe FixtureFormat + fixtureBytes.length shouldBe rustCoreLength + 1 + (fixtureBytes(rustCoreLength) & 0xff) shouldBe terminalMode + + val parsed = parseFixtureEnvelope(fixtureBytes, rustCoreLength, terminalMode).get + parsed.m shouldBe fixtureValue[Int]("m") + parsed.k shouldBe fixtureValue[Int]("k") + parsed.prefix.size shouldBe fixtureValue[Int]("prefix_count") + parsed.suffixHeaders.size shouldBe fixtureValue[Int]("suffix_count") + parsed.suffixTail.size shouldBe fixtureValue[Int]("suffix_tail_count") + Base16.encode(parsed.suffixHead.header.extensionRoot) shouldBe + fixtureValue[String]("extension_root") + parsed.suffixHead.interlinks shouldBe + Seq(deterministicId(0x11), deterministicId(0x22)) + parsed.prefix.head.checkInterlinksProof() shouldBe true + parsed.suffixHead.checkInterlinksProof() shouldBe true + parsed.hasValidParams shouldBe true + parsed.isValid shouldBe true + } + + property("the complete fixture rejects an extension-root mutation") { + val mutated = fixtureBytes.clone() + val extensionRoot = Base16.decode(fixtureValue[String]("extension_root")).get + val rootOffset = + singleSubsliceOffset(mutated, suffixHeadRange(mutated), extensionRoot) + mutated(rootOffset) = (mutated(rootOffset) ^ 1).toByte + + parseFixtureEnvelope(mutated, rustCoreLength, terminalMode).isFailure shouldBe true + } + + property("the complete fixture rejects a disclosed-interlink mutation") { + val mutated = fixtureBytes.clone() + val interlink = Array.fill(32)(0x22.toByte) + val interlinkOffset = + singleSubsliceOffset(mutated, suffixHeadRange(mutated), interlink) + mutated(interlinkOffset) = (mutated(interlinkOffset) ^ 1).toByte + + parseFixtureEnvelope(mutated, rustCoreLength, terminalMode).isFailure shouldBe true + } + + property("the complete fixture rejects an m mutation") { + val mutated = fixtureBytes.clone() + mutated(0) shouldBe 1.toByte + mutated(0) = 0 + + parseFixtureEnvelope(mutated, rustCoreLength, terminalMode).isFailure shouldBe true + } + + property("the complete fixture rejects a k mutation") { + val mutated = fixtureBytes.clone() + mutated.take(2) shouldBe Array[Byte](1, 2) + mutated(1) = 1 + + parseFixtureEnvelope(mutated, rustCoreLength, terminalMode).isFailure shouldBe true + } + + property("the complete fixture rejects a nested header-length mutation") { + val mutated = fixtureBytes.clone() + val nestedSizeOffset = suffixHeadRange(mutated).start + readFixtureVlq(mutated, nestedSizeOffset)._1 shouldBe 218L + mutated(nestedSizeOffset) shouldBe 0xda.toByte + mutated(nestedSizeOffset) = 0xd9.toByte + + parseFixtureEnvelope(mutated, rustCoreLength, terminalMode).isFailure shouldBe true + } + + property("the complete fixture rejects a missing terminal byte") { + parseFixtureEnvelope(fixtureBytes.dropRight(1), rustCoreLength, terminalMode) + .isFailure shouldBe true + } + + property("the complete fixture rejects an extra terminal byte") { + parseFixtureEnvelope( + fixtureBytes :+ terminalMode.toByte, rustCoreLength, terminalMode) + .isFailure shouldBe true + } + + property("the complete fixture rejects terminal-mode mutations") { + val mutated = fixtureBytes.clone() + mutated(mutated.length - 1) = 1 + parseFixtureEnvelope(mutated, rustCoreLength, terminalMode).isFailure shouldBe true + + mutated(mutated.length - 1) = 2 + parseFixtureEnvelope(mutated, rustCoreLength, 2).isFailure shouldBe true + } +} diff --git a/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/PoPowHeaderSpec.scala b/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/PoPowHeaderSpec.scala index 99172ff4b5..8e4911ca94 100644 --- a/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/PoPowHeaderSpec.scala +++ b/ergo-core/src/test/scala/org/ergoplatform/modifiers/history/PoPowHeaderSpec.scala @@ -1,25 +1,447 @@ package org.ergoplatform.modifiers.history +import io.circe.{Decoder, HCursor} +import org.ergoplatform.modifiers.history.extension.ExtensionCandidate import org.ergoplatform.modifiers.history.popow.NipopowAlgos +import org.ergoplatform.modifiers.history.popow.PoPowHeader import org.ergoplatform.modifiers.history.popow.PoPowHeader.checkInterlinksProof +import org.ergoplatform.modifiers.history.popow.PoPowHeaderSerializer import org.ergoplatform.utils.ErgoCorePropertyTest +import org.ergoplatform.utils.generators.ErgoCoreGenerators.defaultHeaderGen import org.scalacheck.Gen +import scorex.crypto.authds.merkle.BatchMerkleProof +import scorex.crypto.hash.Digest32 +import scorex.util.serialization.VLQByteBufferWriter +import scorex.util.{ByteArrayBuilder, ModifierId, bytesToId, idToBytes} +import scorex.util.encode.Base16 + +import java.nio.ByteBuffer +import java.security.MessageDigest +import scala.io.Source class PoPowHeaderSpec extends ErgoCorePropertyTest { import org.ergoplatform.utils.generators.CoreObjectGenerators._ import org.ergoplatform.utils.ErgoCoreTestConstants._ + private def deterministicId(value: Byte): ModifierId = bytesToId(Array.fill(32)(value)) + + private val MaxHeaderBytes = PoPowHeaderSerializer.MaxHeaderBytes + private val MaxInterlinks = PoPowHeaderSerializer.MaxInterlinks + private val MaxMerkleProofBytes = PoPowHeaderSerializer.MaxMerkleProofBytes + + private def resourceCursor(resource: String): HCursor = { + val stream = Option(getClass.getClassLoader.getResourceAsStream(resource)) + .getOrElse(throw new IllegalArgumentException(s"Missing resource: $resource")) + val source = Source.fromInputStream(stream, "UTF-8") + val text = try source.mkString finally source.close() + io.circe.parser.parse(text).fold(error => throw error, value => value.hcursor) + } + + private def fixtureValue[A: Decoder](fixture: HCursor, field: String): A = + fixture.get[A](field).fold(error => throw error, value => value) + + private lazy val sampleHeader: PoPowHeader = { + val fixture = resourceCursor("nipopow-full-root-mixed-popow-header.json") + val bytes = Base16.decode(fixtureValue[String](fixture, "bytes_hex")).get + PoPowHeaderSerializer.parseBytes(bytes) + } + + private def serializeWithNestedPayloads(value: PoPowHeader, + headerPayload: Array[Byte], + proofPayload: Array[Byte]): Array[Byte] = { + writerBytes { writer => + writer.putUInt(headerPayload.length.toLong) + writer.putBytes(headerPayload) + writer.putUInt(value.interlinks.length.toLong) + value.interlinks.foreach(id => writer.putBytes(idToBytes(id))) + writer.putUInt(proofPayload.length.toLong) + writer.putBytes(proofPayload) + } + } + + private def writerBytes(write: VLQByteBufferWriter => Unit): Array[Byte] = { + val writer = new VLQByteBufferWriter(new ByteArrayBuilder) + write(writer) + writer.result().toBytes + } + + private def minimalPoPowHeader(proofPayload: Array[Byte]): Array[Byte] = { + val headerPayload = sampleHeader.header.bytes + writerBytes { writer => + writer.putUInt(headerPayload.length.toLong) + writer.putBytes(headerPayload) + writer.putUInt(0) + writer.putUInt(proofPayload.length.toLong) + writer.putBytes(proofPayload) + } + } + + private def assertParseFailureContains(bytes: Array[Byte], expected: String): Unit = { + val failure = PoPowHeaderSerializer.parseBytesTry(bytes).failed.get + failure.toString should include(expected) + } + + private def intBytes(value: Int): Array[Byte] = ByteBuffer.allocate(4).putInt(value).array() + + private def merkleProofPayload(indices: Seq[Int], proofCount: Int): Array[Byte] = { + val serializedIndices: Array[Byte] = indices.iterator + .flatMap(index => (intBytes(index) ++ Array.fill[Byte](32)(1)).iterator) + .toArray + + intBytes(indices.size) ++ + intBytes(proofCount) ++ + serializedIndices ++ + Array.fill[Byte](proofCount * 33)(0) + } + + private def singletonProofPayload(depth: Int): Array[Byte] = + merkleProofPayload(Seq(0), depth) + + private def mixedExtension(interlinks: Seq[ModifierId]): ExtensionCandidate = { + nipopowAlgos.interlinksToExtension(interlinks) ++ ExtensionCandidate(Seq( + Array[Byte](2, 0) -> Array[Byte](1) + )) + } + property("Check interlinks proof should be true") { forAll(Gen.nonEmptyListOf(modifierIdGen)) { interlinks => - val interlinksProof = NipopowAlgos.proofForInterlinkVector(nipopowAlgos.interlinksToExtension(interlinks)).get - checkInterlinksProof(interlinks, interlinksProof) shouldBe true + val extension = nipopowAlgos.interlinksToExtension(interlinks) + val interlinksProof = NipopowAlgos.proofForInterlinkVector(extension).get + checkInterlinksProof(interlinks, interlinksProof, extension.digest) shouldBe true } } property("Check invalid interlinks proof should be false") { forAll(Gen.nonEmptyListOf(modifierIdGen), Gen.nonEmptyListOf(modifierIdGen)) { (interlinks1, interlinks2) => - val interlinksProof = NipopowAlgos.proofForInterlinkVector(nipopowAlgos.interlinksToExtension(interlinks2)).get - checkInterlinksProof(interlinks1, interlinksProof) shouldBe false + val extension = nipopowAlgos.interlinksToExtension(interlinks2) + val interlinksProof = NipopowAlgos.proofForInterlinkVector(extension).get + checkInterlinksProof(interlinks1, interlinksProof, extension.digest) shouldBe false + } + } + + property("a mixed-extension interlinks proof is accepted against the complete header root") { + val interlinks = Seq(deterministicId(1), deterministicId(2)) + val extension = mixedExtension(interlinks) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + + proof.valid(extension.digest) shouldBe true + proof.valid(extension.interlinksDigest) shouldBe false + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), interlinks, proof) + .checkInterlinksProof() shouldBe true + } + } + + property("a one-byte header extension root mutation is rejected") { + val interlinks = Seq(deterministicId(1), deterministicId(2)) + val extension = mixedExtension(interlinks) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + val wrongRootBytes = extension.digest.clone() + wrongRootBytes(0) = (wrongRootBytes(0) ^ 1).toByte + val wrongRoot = Digest32 @@ wrongRootBytes + + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = wrongRoot), interlinks, proof) + .checkInterlinksProof() shouldBe false + } + } + + property("an interlink mutation retaining the original full-root proof is rejected") { + val interlinks = Seq(deterministicId(1), deterministicId(2)) + val extension = mixedExtension(interlinks) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + val mutatedInterlinks = interlinks.updated(1, deterministicId(3)) + + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), mutatedInterlinks, proof) + .checkInterlinksProof() shouldBe false + } + } + + property("an incomplete interlink disclosure is rejected even when it proves the full root") { + val interlinks = Seq(deterministicId(1), deterministicId(2)) + val extension = mixedExtension(interlinks) + val interlinkKeys = NipopowAlgos.packInterlinks(interlinks).map(_._1) + val incompleteProof = extension.batchProofFor(interlinkKeys.head).get + + incompleteProof.valid(extension.digest) shouldBe true + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), interlinks, incompleteProof) + .checkInterlinksProof() shouldBe false + } + } + + property("an extra disclosed extension leaf is rejected even when it proves the full root") { + val interlinks = Seq(deterministicId(1), deterministicId(2)) + val extension = mixedExtension(interlinks) + val allKeys = extension.fields.map(_._1) + val overcompleteProof = extension.batchProofFor(allKeys: _*).get + + overcompleteProof.valid(extension.digest) shouldBe true + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), interlinks, overcompleteProof) + .checkInterlinksProof() shouldBe false + } + } + + property("an interlinks-only proof is rejected under a mixed extension root") { + val interlinks = Seq(deterministicId(1), deterministicId(2)) + val interlinksOnlyExtension = nipopowAlgos.interlinksToExtension(interlinks) + val mixed = mixedExtension(interlinks) + val legacyProof = NipopowAlgos.proofForInterlinkVector(interlinksOnlyExtension).get + + legacyProof.valid(interlinksOnlyExtension.digest) shouldBe true + legacyProof.valid(mixed.digest) shouldBe false + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = mixed.digest), interlinks, legacyProof) + .checkInterlinksProof() shouldBe false + } + } + + property("a zero-length source run is rejected after unpacking") { + val canonicalFields = NipopowAlgos.packInterlinks(Seq(deterministicId(1), deterministicId(2))) + val zeroLengthValue = canonicalFields.head._2.clone() + zeroLengthValue(0) = 0 + val malformedFields = (canonicalFields.head._1 -> zeroLengthValue) +: canonicalFields.tail + val extension = ExtensionCandidate(malformedFields) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + val unpacked = NipopowAlgos.unpackInterlinks(malformedFields).get + + unpacked shouldBe Seq(deterministicId(2)) + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), unpacked, proof) + .checkInterlinksProof() shouldBe false + } + } + + property("a displaced source run-start key is rejected after unpacking") { + val interlinks = Seq(deterministicId(1), deterministicId(2)) + val canonicalFields = NipopowAlgos.packInterlinks(interlinks) + val displacedKey = canonicalFields(1)._1.clone() + displacedKey(1) = 42 + val malformedFields = Seq(canonicalFields.head, displacedKey -> canonicalFields(1)._2) + val extension = ExtensionCandidate(malformedFields) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + val unpacked = NipopowAlgos.unpackInterlinks(malformedFields).get + + unpacked shouldBe interlinks + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), unpacked, proof) + .checkInterlinksProof() shouldBe false + } + } + + property("the cross-runtime full-root fixture round-trips and rejects mutations") { + val fixture = resourceCursor("nipopow-full-root-mixed-popow-header.json") + val bytes = Base16.decode(fixtureValue[String](fixture, "bytes_hex")).get + + bytes.length shouldBe fixtureValue[Int](fixture, "length") + Base16.encode(MessageDigest.getInstance("SHA-256").digest(bytes)) shouldBe + fixtureValue[String](fixture, "sha256") + + val parsed = PoPowHeaderSerializer.parseBytes(bytes) + PoPowHeaderSerializer.toBytes(parsed) shouldBe bytes + Base16.encode(parsed.header.extensionRoot) shouldBe + fixtureValue[String](fixture, "extension_root") + parsed.checkInterlinksProof() shouldBe true + + val wrongRootBytes = parsed.header.extensionRoot.clone() + wrongRootBytes(0) = (wrongRootBytes(0) ^ 1).toByte + parsed.copy(header = parsed.header.copy(extensionRoot = Digest32 @@ wrongRootBytes)) + .checkInterlinksProof() shouldBe false + + parsed.copy(interlinks = parsed.interlinks.updated(1, deterministicId(3))) + .checkInterlinksProof() shouldBe false + } + + property("a nested header payload accepts trailing padding") { + val headerPayload = sampleHeader.header.bytes :+ 0x7f.toByte + val proofPayload = PoPowHeaderSerializer.merkleProofSerializer.serialize(sampleHeader.interlinksProof) + val bytes = serializeWithNestedPayloads(sampleHeader, headerPayload, proofPayload) + + PoPowHeaderSerializer.parseBytes(bytes) shouldBe sampleHeader + } + + property("a nested Merkle proof payload rejects trailing padding") { + val headerPayload = sampleHeader.header.bytes + val proofPayload = + PoPowHeaderSerializer.merkleProofSerializer.serialize(sampleHeader.interlinksProof) :+ 0x7f.toByte + val bytes = serializeWithNestedPayloads(sampleHeader, headerPayload, proofPayload) + + assertParseFailureContains(bytes, "Merkle proof counts") + } + + property("PoPowHeader rejects an oversized nested header before reading its payload") { + val bytes = writerBytes(_.putUInt(MaxHeaderBytes + 1L)) + + assertParseFailureContains(bytes, "header length") + } + + property("PoPowHeader rejects an oversized interlink count before reading ids") { + val headerPayload = sampleHeader.header.bytes + val bytes = writerBytes { writer => + writer.putUInt(headerPayload.length.toLong) + writer.putBytes(headerPayload) + writer.putUInt(MaxInterlinks + 1L) + } + + assertParseFailureContains(bytes, "interlink count") + } + + property("PoPowHeader rejects an oversized Merkle proof before reading its payload") { + val headerPayload = sampleHeader.header.bytes + val bytes = writerBytes { writer => + writer.putUInt(headerPayload.length.toLong) + writer.putBytes(headerPayload) + writer.putUInt(0) + writer.putUInt(MaxMerkleProofBytes + 1L) + } + + assertParseFailureContains(bytes, "Merkle proof length") + } + + property("PoPowHeader rejects an index count that cannot fit its proof payload") { + val proofPayload = intBytes(1) ++ intBytes(0) + + assertParseFailureContains(minimalPoPowHeader(proofPayload), "Merkle proof counts") + } + + property("PoPowHeader rejects a proof-node count that cannot fit its proof payload") { + val proofPayload = intBytes(0) ++ intBytes(1) + + assertParseFailureContains(minimalPoPowHeader(proofPayload), "Merkle proof counts") + } + + property("PoPowHeader rejects a singleton proof deeper than the extension key space") { + val impossibleDepth = java.lang.Byte.SIZE * 2 + 1 + + assertParseFailureContains( + minimalPoPowHeader(singletonProofPayload(impossibleDepth)), + "Merkle proof structure" + ) + } + + property("PoPowHeader accepts a singleton proof at the extension key-space depth") { + val maximumDepth = java.lang.Byte.SIZE * 2 + + PoPowHeaderSerializer.parseBytes( + minimalPoPowHeader(singletonProofPayload(maximumDepth)) + ).interlinksProof.proofs.size shouldBe maximumDepth + } + + property("PoPowHeader rejects a Merkle index outside the extension key space") { + val firstInvalidIndex = 1 << PoPowHeaderSerializer.MaxMerkleProofDepth + + assertParseFailureContains( + minimalPoPowHeader(merkleProofPayload(Seq(firstInvalidIndex), 0)), + "Merkle proof structure" + ) + } + + property("PoPowHeader rejects duplicate Merkle indices") { + assertParseFailureContains( + minimalPoPowHeader(merkleProofPayload(Seq(0, 0), 0)), + "Merkle proof structure" + ) + } + + property("PoPowHeader rejects extreme Merkle counts with checked arithmetic") { + val proofPayload = intBytes(Int.MaxValue) ++ intBytes(Int.MaxValue) + + assertParseFailureContains(minimalPoPowHeader(proofPayload), "Merkle proof counts") + } + + property("empty interlinks proof is accepted for genesis") { + val extension = nipopowAlgos.interlinksToExtension(Seq.empty) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 1, extensionRoot = extension.digest), Seq.empty, proof) + .checkInterlinksProof() shouldBe true + } + } + + property("empty interlinks proof is rejected for non-genesis headers") { + val extension = nipopowAlgos.interlinksToExtension(Seq.empty) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), Seq.empty, proof) + .checkInterlinksProof() shouldBe false + } + } + + property("Merkle library validation exceptions reject the interlinks proof") { + val interlinks = Seq(deterministicId(1), deterministicId(2)) + val extension = mixedExtension(interlinks) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + proof.proofs should not be empty + val malformedProof = BatchMerkleProof[Digest32]( + proof.indices, + (null.asInstanceOf[Digest32] -> proof.proofs.head._2) +: proof.proofs.tail + )(org.ergoplatform.settings.Algos.hash) + + checkInterlinksProof(interlinks, malformedProof, extension.digest) shouldBe false + } + + property("missing Merkle proof nodes reject the interlinks proof") { + val interlinks = Seq(deterministicId(1), deterministicId(2)) + val extension = mixedExtension(interlinks) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + val incompleteProof = BatchMerkleProof[Digest32]( + proof.indices, + Seq.empty + )(org.ergoplatform.settings.Algos.hash) + + checkInterlinksProof(interlinks, incompleteProof, extension.digest) shouldBe false + } + + property("a canonical run of 255 identical interlinks is accepted") { + val interlinks = Seq.fill(255)(deterministicId(1)) + val extension = nipopowAlgos.interlinksToExtension(interlinks) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), interlinks, proof) + .checkInterlinksProof() shouldBe true + } + } + + property("a run of 256 identical interlinks is rejected") { + val interlinks = Seq.fill(256)(deterministicId(1)) + val extension = nipopowAlgos.interlinksToExtension(interlinks) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), interlinks, proof) + .checkInterlinksProof() shouldBe false + } + } + + property("a new interlink run beginning at position 256 is rejected") { + val first = deterministicId(1) + val second = deterministicId(2) + val third = deterministicId(3) + val interlinks = Seq.fill(255)(first) ++ Seq(second, third) + val extension = nipopowAlgos.interlinksToExtension(interlinks) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), interlinks, proof) + .checkInterlinksProof() shouldBe false + } + } + + property("a closed interlink id cannot reopen in a later run") { + val first = deterministicId(1) + val second = deterministicId(2) + val interlinks = Seq(first, second, first) + val extension = nipopowAlgos.interlinksToExtension(interlinks) + val proof = NipopowAlgos.proofForInterlinkVector(extension).get + + forAll(defaultHeaderGen) { header => + PoPowHeader(header.copy(height = 2, extensionRoot = extension.digest), interlinks, proof) + .checkInterlinksProof() shouldBe false } } } diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 80af5fa3ad..5a5f8fc75e 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "6.0.4" + version: "6.0.5" title: Ergo Node API description: API docs for Ergo Node. Models are shared between all Ergo products contact: diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 4bbea9b00a..bc8db13f0e 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -446,7 +446,7 @@ scorex { nodeName = "ergo-node" # Network protocol version to be sent in handshakes - appVersion = 6.0.4 + appVersion = 6.0.5 # Network agent name. May contain information about client code # stack, starting from core code-base up to the end graphical interface. diff --git a/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProverWithDbAlgs.scala b/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProverWithDbAlgs.scala index d4bffa263d..94dc4924fa 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProverWithDbAlgs.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/popow/NipopowProverWithDbAlgs.scala @@ -29,7 +29,6 @@ object NipopowProverWithDbAlgs { val k = params.k val m = params.m - require(params.k >= 1, s"$k < 1") require(histReader.headersHeight >= k + m, s"Can not prove chain of size < ${k + m}") def linksWithIndexes(header: PoPowHeader): Seq[(ModifierId, Int)] = header.interlinks.tail.reverse.zipWithIndex diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index ced63d1e8a..a5d18a6f8a 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -1066,7 +1066,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, val msg = Message(NipopowProofSpec, Right(proofBytes), None) networkControllerRef ! SendToNetwork(msg, SendToPeer(peer)) case None => - log.warn("No Nipopow Proof available") + log.warn("No cached Nipopow proof available") } } else { // for now, we are serving proofs for concrete params only diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala index 9f7a45f0fe..fbe6ca5e6e 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala @@ -587,10 +587,10 @@ trait ErgoHistoryReader } /** - * @return serialized NiPoPoW proof store in database + * @return serialized NiPoPoW proof stored under the current P2P cache key */ def readPopowProofBytesFromDb(): Option[Array[Byte]] = { - historyStorage.getIndex(NipopowSnapshotHeightKey) + historyStorage.getIndex(NipopowProofV2Key) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/PopowProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/PopowProcessor.scala index 59922347a3..86b1260be3 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/PopowProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/PopowProcessor.scala @@ -43,7 +43,14 @@ trait PopowProcessor extends BasicReaders with ScorexLogging { private lazy val nipopowVerifier = new NipopowVerifier(chainSettings.genesisId.orElse(bestHeaderIdAtHeight(ErgoHistoryUtils.GenesisHeight))) - protected val NipopowSnapshotHeightKey: ByteArrayWrapper = ByteArrayWrapper(Array.fill(HashLength)(50: Byte)) + private[history] val LegacyNipopowSnapshotKey: ByteArrayWrapper = + ByteArrayWrapper(Array.fill(HashLength)(50: Byte)) + + private[history] val NipopowProofV2Key: ByteArrayWrapper = + ByteArrayWrapper(Array.fill(HashLength)(51: Byte)) + + // Periodic snapshot production and P2P reads use V2. The legacy row remains untouched. + protected val NipopowSnapshotHeightKey: ByteArrayWrapper = NipopowProofV2Key /** * Minimal superchain length ('m' in KMZ17 paper) value used in NiPoPoW proofs for bootstrapping @@ -107,8 +114,9 @@ trait PopowProcessor extends BasicReaders with ScorexLogging { * @return PoPow proof if success, Failure instance otherwise */ def popowProof(m: Int, k: Int, headerIdOpt: Option[ModifierId]): Try[NipopowProof] = { - val proofParams = PoPowParams(m, k, continuous = true) - NipopowProverWithDbAlgs.prove(historyReader, headerIdOpt = headerIdOpt, chainSettings)(proofParams) + PoPowParams(m, k, continuous = true).flatMap { proofParams => + NipopowProverWithDbAlgs.prove(historyReader, headerIdOpt = headerIdOpt, chainSettings)(proofParams) + } } /** diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index 6e58782dfe..a919afb76a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -323,7 +323,8 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, case _ => None } - loop(waitMinutes = 0).getOrElse(settings.nodeSettings.minimalFeeAmount) + val recommendedFee = loop(waitMinutes = 0).getOrElse(settings.nodeSettings.minimalFeeAmount) + math.max(recommendedFee, settings.nodeSettings.minimalFeeAmount) } /** @@ -346,8 +347,9 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, // Time since statistics measurement interval (needed to calculate average tx rate) val elapsed = System.currentTimeMillis() - stats.startMeasurement - if (stats.takenTxns != 0) { - elapsed * posInPool / stats.takenTxns + val cappedElapsed = math.max(0L, math.min(elapsed, MemPoolStatistics.measurementIntervalMsec.toLong)) + if (stats.takenTxns > 0) { + cappedElapsed * posInPool / stats.takenTxns } else { 0 } diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala index 03b91e19ca..17e2a7e36b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala @@ -324,7 +324,7 @@ trait ErgoWalletSupport extends ScorexLogging { require(outputs.forall(_.additionalTokens.forall(_._2 > 0)), "Non-positive asset value") val assetIssueBox = outputs - .zip(requests) + .zip(requestsWithoutBurnTokens) .filter(_._2.isInstanceOf[AssetIssueRequest]) .map(_._1) .headOption diff --git a/src/main/scala/scorex/core/network/NetworkController.scala b/src/main/scala/scorex/core/network/NetworkController.scala index 8db3500db9..0cec487f35 100644 --- a/src/main/scala/scorex/core/network/NetworkController.scala +++ b/src/main/scala/scorex/core/network/NetworkController.scala @@ -163,11 +163,11 @@ class NetworkController(ergoSettings: ErgoSettings, peerManagerRef ! PeerManager.ReceivableMessages.Penalize(peerAddress, penaltyType) case Blacklisted(peerAddress) => - connections.get(peerAddress).foreach { peer => - connections = connections.filterNot { case (address, _) => // clear all connections related to banned peer ip - Option(peer.connectionId.remoteAddress.getAddress).exists(Option(address.getAddress).contains(_)) - } - peer.handlerRef ! CloseConnection + Option(peerAddress.getAddress).foreach { blacklistedIp => + val peersToClose = connections.valuesIterator.filter { peer => + Option(peer.connectionId.remoteAddress.getAddress).contains(blacklistedIp) + }.toSeq + peersToClose.foreach(_.handlerRef ! CloseConnection) } } diff --git a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala index 8c32487397..bd6a573bed 100644 --- a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala +++ b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala @@ -4,14 +4,19 @@ import akka.actor.{Actor, ActorRef, Cancellable, Props, SupervisorStrategy} import akka.io.Tcp import akka.io.Tcp._ import akka.util.{ByteString, CompactByteString} -import org.ergoplatform.network.{Handshake, HandshakeSerializer, PeerSpec, Version} import org.ergoplatform.network.Version.Eip37ForkVersion -import scorex.core.app.ScorexContext -import scorex.core.network.NetworkController.ReceivableMessages.{Handshaked, PenalizePeer} -import scorex.core.network.PeerConnectionHandler.ReceivableMessages +import org.ergoplatform.network.{Handshake, HandshakeSerializer, PeerSpec, Version} +import org.ergoplatform.network.message.MessageConstants.{ + ChecksumLength, + HeaderLength, + MaxMessageSize +} import org.ergoplatform.network.message.MessageSerializer import org.ergoplatform.network.peer.{PeerInfo, PenaltyType} import org.ergoplatform.settings.ScorexSettings +import scorex.core.app.ScorexContext +import scorex.core.network.NetworkController.ReceivableMessages.{Handshaked, PenalizePeer} +import scorex.core.network.PeerConnectionHandler.ReceivableMessages import scorex.util.ScorexLogging import scala.annotation.tailrec @@ -27,6 +32,7 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, extends Actor with ScorexLogging { import PeerConnectionHandler.ReceivableMessages._ + import PeerConnectionHandler.{MaxBufferedOutboundBytes, MaxBufferedOutboundMessages} private val networkSettings = scorexSettings.network private val connection = connectionDescription.connection @@ -48,6 +54,8 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, private var outMessagesBuffer: TreeMap[Long, ByteString] = TreeMap.empty + private var outMessagesBufferBytes: Long = 0L + private var outMessagesCounter: Long = 0 override def preStart: Unit = { @@ -179,7 +187,10 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, writeFirst() case ReceivableMessages.Ack(id) => - outMessagesBuffer -= id + outMessagesBuffer.get(id).foreach { msg => + outMessagesBuffer -= id + outMessagesBufferBytes -= msg.length + } if (outMessagesBuffer.nonEmpty){ writeFirst() } else { @@ -226,7 +237,22 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, } private def buffer(id: Long, msg: ByteString): Unit = { - outMessagesBuffer += id -> msg + val previousMessage = outMessagesBuffer.get(id) + val previousLength = previousMessage.fold(0)(_.length) + val candidateBytes = outMessagesBufferBytes - previousLength + msg.length + val candidateMessages = outMessagesBuffer.size + previousMessage.fold(1)(_ => 0) + if (candidateBytes > MaxBufferedOutboundBytes || + candidateMessages > MaxBufferedOutboundMessages) { + log.warn(s"Buffered outbound data for $connectionId would exceed its limit " + + s"($candidateMessages messages, $candidateBytes bytes), aborting the connection") + outMessagesBuffer = TreeMap.empty + outMessagesBufferBytes = 0L + connection ! Abort + context.stop(self) + } else { + outMessagesBuffer += id -> msg + outMessagesBufferBytes = candidateBytes + } } private def writeFirst(): Unit = { @@ -259,6 +285,14 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, object PeerConnectionHandler { + // Keep one maximum serialized frame per peer. Backpressured snapshot transfers + // retry instead of retaining their entire application-level in-flight window. + private[network] val MaxBufferedOutboundBytes: Long = + MaxMessageSize.toLong + HeaderLength + ChecksumLength + + // Independently bound collection overhead from small messages. + private[network] val MaxBufferedOutboundMessages: Int = 64 + object ReceivableMessages { case object HandshakeTimeout diff --git a/src/test/scala/org/ergoplatform/http/routes/NipopowApiRoutesSpec.scala b/src/test/scala/org/ergoplatform/http/routes/NipopowApiRoutesSpec.scala index cfc95a8337..205aedc0bf 100644 --- a/src/test/scala/org/ergoplatform/http/routes/NipopowApiRoutesSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/NipopowApiRoutesSpec.scala @@ -35,6 +35,12 @@ class NipopowApiRoutesSpec extends AnyFlatSpec } } + it should "reject proof request when minimum and suffix length overflow" in { + Get(s"/nipopow/proof/${Int.MaxValue}/1") ~> route ~> check { + status shouldBe StatusCodes.BadRequest + } + } + it should "proof request with missing headerId" in { Get("/nipopow/proof/1/1/05bf63aa1ecfc9f4e3fadc993f87b33edb4d58e151c1891816d734dd5a0e2e09") ~> route ~> check { status shouldBe StatusCodes.BadRequest diff --git a/src/test/scala/org/ergoplatform/local/NipopowVerifierSpec.scala b/src/test/scala/org/ergoplatform/local/NipopowVerifierSpec.scala index 1216d77244..4d8e153bc6 100644 --- a/src/test/scala/org/ergoplatform/local/NipopowVerifierSpec.scala +++ b/src/test/scala/org/ergoplatform/local/NipopowVerifierSpec.scala @@ -1,5 +1,8 @@ package org.ergoplatform.local +import java.util.concurrent.{CountDownLatch, TimeUnit} +import java.util.concurrent.atomic.AtomicReference + import org.ergoplatform.modifiers.history.popow.{PoPowHeader, PoPowParams} import org.ergoplatform.modifiers.ErgoFullBlock import org.scalatest.matchers.should.Matchers @@ -11,7 +14,7 @@ class NipopowVerifierSpec extends AnyPropSpec with Matchers { import org.ergoplatform.utils.generators.ChainGenerator._ - private val poPowParams = PoPowParams(30, 30, continuous = false) + private val poPowParams = PoPowParams(30, 30, continuous = false).get val toPoPoWChain = (c: Seq[ErgoFullBlock]) => c.map(b => PoPowHeader.fromBlock(b).get) property("processes new proofs") { @@ -43,4 +46,50 @@ class NipopowVerifierSpec extends AnyPropSpec with Matchers { verifier.bestChain.last.id shouldBe longestProof.headersChain.last.id } } + + property("rejects proofs with invalid security parameters") { + val baseChain = genChain(100) + val params = PoPowParams(5, 5, continuous = false).get + val proof = nipopowAlgos.prove(toPoPoWChain(baseChain))(params).get + + Seq( + proof.copy(m = 0), + proof.copy(k = 0), + proof.copy(m = Int.MaxValue, k = 1) + ).foreach { invalidProof => + invalidProof.isValid shouldBe false + an[IllegalArgumentException] should be thrownBy + invalidProof.serializer.toBytes(invalidProof) + + val verifier = new NipopowVerifier(Some(baseChain.head.id)) + verifier.process(invalidProof) shouldBe ValidationError + verifier.bestChain shouldBe empty + } + } + + property("returns when a duplicate invalid proof is processed") { + val baseChain = genChain(100) + val params = PoPowParams(5, 5, continuous = false).get + val invalidProof = nipopowAlgos.prove(toPoPoWChain(baseChain))(params).get.copy(m = 0) + invalidProof.isValid shouldBe false + an[IllegalArgumentException] should be thrownBy + invalidProof.serializer.toBytes(invalidProof) + val verifier = new NipopowVerifier(Some(baseChain.head.id)) + + val firstResult = verifier.process(invalidProof) + val secondResult = new AtomicReference[NipopowProofVerificationResult]() + val completed = new CountDownLatch(1) + val worker = new Thread(new Runnable { + override def run(): Unit = + try secondResult.set(verifier.process(invalidProof)) + finally completed.countDown() + }) + worker.setDaemon(true) + worker.start() + + completed.await(2, TimeUnit.SECONDS) shouldBe true + firstResult shouldBe ValidationError + secondResult.get() shouldBe ValidationError + verifier.bestChain shouldBe empty + } } diff --git a/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosSpec.scala b/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosSpec.scala index 66d903e79b..7497100d41 100644 --- a/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosSpec.scala +++ b/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosSpec.scala @@ -1,7 +1,11 @@ package org.ergoplatform.modifiers.history +import java.util.concurrent.{CountDownLatch, TimeUnit} +import java.util.concurrent.atomic.AtomicReference + import org.ergoplatform.modifiers.history.popow.{NipopowAlgos, NipopowProof, PoPowHeader, PoPowParams} import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.modifiers.history.header.Header import org.scalacheck.Gen import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec @@ -12,11 +16,59 @@ class PoPowAlgosSpec extends AnyPropSpec with Matchers { import org.ergoplatform.utils.generators.CoreObjectGenerators._ import org.ergoplatform.utils.ErgoCoreTestConstants._ - private val poPowParams = PoPowParams(30, 30, continuous = false) + private val poPowParams = PoPowParams(30, 30, continuous = false).get private val ChainLength = 10 private def toPoPoWChain = (c: Seq[ErgoFullBlock]) => c.map(b => PoPowHeader.fromBlock(b).get) + property("PoPowParams rejects invalid minimum chain lengths") { + PoPowParams.isValid(0, 1) shouldBe false + PoPowParams.isValid(1, 0) shouldBe false + PoPowParams.isValid(Int.MaxValue, 1) shouldBe false + + PoPowParams(0, 1, continuous = false) shouldBe 'failure + PoPowParams(1, 0, continuous = false) shouldBe 'failure + PoPowParams(Int.MaxValue, 1, continuous = false) shouldBe 'failure + + PoPowParams.isValid(PoPowParams.MaxProofElements, PoPowParams.MaxProofElements) shouldBe true + PoPowParams(1, 1, continuous = false).get.minChainLength shouldBe 2 + } + + property("bestArg rejects a non-positive security parameter without looping") { + val algos = nipopowAlgos + val completed = new CountDownLatch(1) + val error = new AtomicReference[Throwable]() + val worker = new Thread(new Runnable { + override def run(): Unit = + try { + algos.bestArg(Seq.empty)(0) + } catch { + case t: Throwable => error.set(t) + } finally { + completed.countDown() + } + }) + worker.setDaemon(true) + worker.start() + + completed.await(2, TimeUnit.SECONDS) shouldBe true + error.get() shouldBe a[IllegalArgumentException] + } + + private def validProof(m: Int = 1, k: Int = 1): NipopowProof = { + val chain = toPoPoWChain(genChain(m + k + 4)) + nipopowAlgos.prove(chain)(PoPowParams(m, k, continuous = false).get).get + } + + private class CountingNipopowAlgos extends NipopowAlgos(nipopowAlgos.chainSettings) { + var bestArgCalls: Int = 0 + + override def bestArg(chain: Seq[Header])(m: Int): Int = { + bestArgCalls += 1 + super.bestArg(chain)(m) + } + } + property("updateInterlinks") { val chain = genChain(ChainLength) val genesis = chain.head @@ -144,7 +196,7 @@ class PoPowAlgosSpec extends AnyPropSpec with Matchers { } property("isBetterThan - a disconnected prefix chain should not win") { - val smallPoPowParams = PoPowParams(50, 1, continuous = false) + val smallPoPowParams = PoPowParams(50, 1, continuous = false).get val size = 100 val chain = toPoPoWChain(genChain(size)) val proof = nipopowAlgos.prove(chain)(smallPoPowParams).get @@ -158,7 +210,7 @@ class PoPowAlgosSpec extends AnyPropSpec with Matchers { } property("hasValidConnections - ensures a connected prefix chain") { - val smallPoPowParams = PoPowParams(5, 5, continuous = false) + val smallPoPowParams = PoPowParams(5, 5, continuous = false).get val sizes = Seq(100, 200) sizes.foreach { size => val chain = toPoPoWChain(genChain(size)) @@ -172,7 +224,7 @@ class PoPowAlgosSpec extends AnyPropSpec with Matchers { } property("hasValidConnections - ensures a connected suffix chain") { - val smallPoPowParams = PoPowParams(5, 5, continuous = false) + val smallPoPowParams = PoPowParams(5, 5, continuous = false).get val sizes = Seq(100, 200) sizes.foreach { size => @@ -192,4 +244,87 @@ class PoPowAlgosSpec extends AnyPropSpec with Matchers { NipopowProof(nipopowAlgos, 0, 0, prefix, suffix.head, suffix.tail.map(_.header), continuous = false).hasValidConnections shouldBe false } + property("PoPowParams rejects invalid m") { + Seq(-1, 0, 20001).foreach { invalidM => + withClue(s"m=$invalidM") { + PoPowParams(invalidM, 1, continuous = false).isFailure shouldBe true + } + } + } + + property("PoPowParams rejects invalid k") { + Seq(-1, 0, 20001).foreach { invalidK => + withClue(s"k=$invalidK") { + PoPowParams(1, invalidK, continuous = false).isFailure shouldBe true + } + } + } + + property("PoPowParams accepts the sanity-bound endpoints") { + PoPowParams(1, 1, continuous = false).isSuccess shouldBe true + PoPowParams(20000, 20000, continuous = false).isSuccess shouldBe true + } + + property("NipopowProof rejects invalid m") { + val proof = validProof() + proof.isValid shouldBe true + Seq(-1, 0, 20001).foreach { invalidM => + withClue(s"m=$invalidM") { + proof.copy(m = invalidM).isValid shouldBe false + } + } + } + + property("NipopowProof rejects invalid k") { + val proof = validProof() + proof.isValid shouldBe true + Seq(-1, 0, 20001).foreach { invalidK => + withClue(s"k=$invalidK") { + proof.copy(k = invalidK).isValid shouldBe false + } + } + } + + property("NipopowProof rejects a suffix length different from k") { + val proof = validProof() + proof.isValid shouldBe true + proof.copy(k = 2).isValid shouldBe false + } + + property("NipopowProof exposes parameter and suffix validity") { + val proof = validProof() + proof.hasValidParams shouldBe true + proof.copy(m = 0).hasValidParams shouldBe false + proof.copy(k = 2).hasValidParams shouldBe false + } + + property("isBetterThan rejects unequal m before scoring") { + val chain = toPoPoWChain(genChain(12)) + val left = nipopowAlgos.prove(chain)(PoPowParams(1, 2, continuous = false).get).get + val right = nipopowAlgos.prove(chain)(PoPowParams(2, 2, continuous = false).get).get + left.isValid shouldBe true + right.isValid shouldBe true + val counting = new CountingNipopowAlgos + + left.copy(popowAlgos = counting).isBetterThan(right.copy(popowAlgos = counting)) shouldBe false + counting.bestArgCalls shouldBe 0 + } + + property("isBetterThan rejects unequal k before scoring") { + val chain = toPoPoWChain(genChain(12)) + val left = nipopowAlgos.prove(chain)(PoPowParams(1, 2, continuous = false).get).get + val right = nipopowAlgos.prove(chain)(PoPowParams(1, 3, continuous = false).get).get + left.isValid shouldBe true + right.isValid shouldBe true + val counting = new CountingNipopowAlgos + + left.copy(popowAlgos = counting).isBetterThan(right.copy(popowAlgos = counting)) shouldBe false + counting.bestArgCalls shouldBe 0 + } + + property("maxLevelOf rejects a zero decoded target") { + val nonGenesis = genChain(2).last.header.copy(nBits = 0) + an[IllegalArgumentException] should be thrownBy nipopowAlgos.maxLevelOf(nonGenesis) + } + } diff --git a/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosWithDBSpec.scala b/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosWithDBSpec.scala index c81df767c2..f708e6e966 100644 --- a/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosWithDBSpec.scala +++ b/src/test/scala/org/ergoplatform/modifiers/history/PoPowAlgosWithDBSpec.scala @@ -12,7 +12,7 @@ class PoPowAlgosWithDBSpec extends AnyPropSpec with Matchers { import org.ergoplatform.utils.generators.ChainGenerator._ property("proof(chain) is equivalent to proof(histReader)") { - val poPowParams = PoPowParams(m = 5, k = 6, continuous = false) + val poPowParams = PoPowParams(m = 5, k = 6, continuous = false).get val blocksChain = genChain(3000) val pchain = blocksChain.map(b => PoPowHeader.fromBlock(b).get) val proof0 = nipopowAlgos.prove(pchain)(poPowParams).get @@ -30,7 +30,7 @@ class PoPowAlgosWithDBSpec extends AnyPropSpec with Matchers { } property("proof(histReader) for a header in the past") { - val poPowParams = PoPowParams(5, 6, continuous = false) + val poPowParams = PoPowParams(5, 6, continuous = false).get val blocksChain = genChain(300) val at = 200 diff --git a/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala b/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala index b81c49672b..5ed7f095a9 100644 --- a/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala +++ b/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala @@ -144,32 +144,84 @@ trait NodeViewSynchronizerTests[ST <: ErgoState[ST]] extends AnyPropSpec } } - property("NodeViewSynchronizer: GetNipopowProof") { + property("NodeViewSynchronizer: GetNipopowProof serves and reuses the V2 cache") { withFixture { ctx => import ctx._ - // Generate history chain val emptyHistory = historyGen.sample.get - val prefix = blockStream(None).take(settings.chainSettings.makeSnapshotEvery) - val fullHistory = applyChain(emptyHistory, prefix) + val chain = blockStream(None).take(settings.chainSettings.makeSnapshotEvery) + val history = applyChain(emptyHistory, chain) + val cachedBytes = history.readPopowProofBytesFromDb().get // Broadcast updated history - node ! ChangedHistory(fullHistory) + node ! ChangedHistory(history) // Build and send GetNipopowProofSpec request val spec = GetNipopowProofSpec - val msgBytes = spec.toBytes(NipopowProofData(m = emptyHistory.P2PNipopowProofM, k = emptyHistory.P2PNipopowProofK, headerId = None)) + val msgBytes = spec.toBytes(NipopowProofData( + m = history.P2PNipopowProofM, + k = history.P2PNipopowProofK, + headerId = None + )) node ! Message[NipopowProofData](spec, Left(msgBytes), Option(peer)) - // Listen for NipopowProofSpec response - ncProbe.fishForMessage(5 seconds) { + // Listen for the cached NipopowProofSpec response + val firstResponse = ncProbe.fishForMessage(5 seconds) { case stn: SendToNetwork => stn.message.spec match { case _: NipopowProofSpec.type => true case _ => false } case _: Any => false + }.asInstanceOf[SendToNetwork] + val firstBytes = firstResponse.message.data.get.asInstanceOf[Array[Byte]] + firstBytes.toSeq shouldBe cachedBytes.toSeq + + // A second request must reuse the persisted bytes exactly. + node ! Message[NipopowProofData](spec, Left(msgBytes), Option(peer)) + val secondResponse = ncProbe.fishForMessage(5 seconds) { + case stn: SendToNetwork => stn.message.spec.isInstanceOf[NipopowProofSpec.type] + case _: Any => false + }.asInstanceOf[SendToNetwork] + secondResponse.message.data.get.asInstanceOf[Array[Byte]].toSeq shouldBe firstBytes.toSeq + history.readPopowProofBytesFromDb().get.toSeq shouldBe cachedBytes.toSeq + } + } + + property("NodeViewSynchronizer: GetNipopowProof sends nothing when the V2 cache is missing") { + withFixture { ctx => + import ctx._ + + val emptyHistory = org.ergoplatform.utils.HistoryTestHelpers.generateHistory( + verifyTransactions = true, + StateType.Utxo, + PoPoWBootstrap = false, + blocksToKeep = -1 + ) + val history = applyChain( + emptyHistory, + blockStream(None).take(settings.chainSettings.makeSnapshotEvery / 2) + ) + history.readPopowProofBytesFromDb() shouldBe None + node ! ChangedHistory(history) + ncProbe.receiveWhile(max = 1.second, idle = 200.millis) { case message => message } + + val spec = GetNipopowProofSpec + val msgBytes = spec.toBytes(NipopowProofData( + m = history.P2PNipopowProofM, + k = history.P2PNipopowProofK, + headerId = None + )) + node ! Message[NipopowProofData](spec, Left(msgBytes), Option(peer)) + + val responses = ncProbe.receiveWhile(max = 2.seconds, idle = 500.millis) { + case message => message } + responses.exists { + case stn: SendToNetwork => stn.message.spec.isInstanceOf[NipopowProofSpec.type] + case _ => false + } shouldBe false + history.readPopowProofBytesFromDb() shouldBe None } } diff --git a/src/test/scala/org/ergoplatform/nodeView/history/PopowProcessorSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/PopowProcessorSpecification.scala index d9ace006c3..69959f5d36 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/PopowProcessorSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/PopowProcessorSpecification.scala @@ -1,13 +1,19 @@ package org.ergoplatform.nodeView.history -import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.mining.AutolykosPowScheme +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} import org.ergoplatform.modifiers.history.popow.PoPowHeader import org.ergoplatform.nodeView.state.StateType -import org.ergoplatform.utils.ErgoCorePropertyTest +import org.ergoplatform.settings.NipopowSettings +import org.ergoplatform.utils.{ErgoCorePropertyTest, ErgoNodeTestConstants} +import org.ergoplatform.wallet.utils.FileUtils import scorex.util.ModifierId -class PopowProcessorSpecification extends ErgoCorePropertyTest { +import java.nio.charset.StandardCharsets + +class PopowProcessorSpecification extends ErgoCorePropertyTest with FileUtils { import org.ergoplatform.utils.HistoryTestHelpers._ + import org.ergoplatform.utils.ErgoNodeTestConstants.{settings => baseSettings} import org.ergoplatform.utils.generators.ChainGenerator._ private def genHistory(genesisIdOpt: Option[ModifierId], popowBootstrap: Boolean) = @@ -15,8 +21,141 @@ class PopowProcessorSpecification extends ErgoCorePropertyTest { epochLength = 10000, useLastEpochs = 3, initialDiffOpt = None, genesisIdOpt) .ensuring(_.bestFullBlockOpt.isEmpty) + private def genRealPowHistory(genesisIdOpt: Option[ModifierId], + realPowScheme: AutolykosPowScheme): ErgoHistory = { + val realPowSettings = baseSettings.copy( + directory = createTempDir.getAbsolutePath, + chainSettings = baseSettings.chainSettings.copy(powScheme = realPowScheme, genesisId = genesisIdOpt), + nodeSettings = baseSettings.nodeSettings.copy( + stateType = StateType.Utxo, + verifyTransactions = true, + blocksToKeep = -1, + nipopowSettings = NipopowSettings(nipopowBootstrap = true, p2pNipopows = 1) + ) + ) + ErgoHistory.readOrGenerate(realPowSettings)(null).ensuring(_.bestFullBlockOpt.isEmpty) + } + val toPoPoWChain = (c: Seq[ErgoFullBlock]) => c.map(b => PoPowHeader.fromBlock(b).get) + private val legacySentinel = + "legacy-nipopow-proof-must-not-be-served".getBytes(StandardCharsets.UTF_8) + + property("legacy NiPoPoW cache bytes are ignored") { + val history = genHistory(None, popowBootstrap = false) + try { + history.LegacyNipopowSnapshotKey.data.toSeq shouldBe Seq.fill(32)(50.toByte) + history.NipopowProofV2Key.data.toSeq shouldBe Seq.fill(32)(51.toByte) + history.LegacyNipopowSnapshotKey should not equal history.NipopowProofV2Key + + history.historyStorage.insert( + Array(history.LegacyNipopowSnapshotKey -> legacySentinel), + BlockSection.emptyArray + ).get + + history.readPopowProofBytesFromDb() shouldBe None + history.historyStorage + .getIndex(history.LegacyNipopowSnapshotKey) + .get + .toSeq shouldBe legacySentinel.toSeq + } finally { + history.closeStorage() + } + } + + property("V2 NiPoPoW cache miss remains empty until a scheduled snapshot") { + val history = genHistory(None, popowBootstrap = false) + try { + history.readPopowProofBytesFromDb() shouldBe None + history.readPopowProofBytesFromDb() shouldBe None + } finally { + history.closeStorage() + } + } + + property("V2 NiPoPoW cache hit reuses byte-identical proof without generation") { + val history = genHistory(None, popowBootstrap = false) + val cachedBytes = Array[Byte](2, 4, 6, 8) + try { + history.historyStorage.insert( + Array(history.NipopowProofV2Key -> cachedBytes), + BlockSection.emptyArray + ).get + + history.readPopowProofBytesFromDb().get.toSeq shouldBe cachedBytes.toSeq + history.readPopowProofBytesFromDb().get.toSeq shouldBe cachedBytes.toSeq + } finally { + history.closeStorage() + } + } + + property("V2 NiPoPoW cache miss never falls back to legacy bytes") { + val history = genHistory(None, popowBootstrap = false) + try { + history.historyStorage.insert( + Array(history.LegacyNipopowSnapshotKey -> legacySentinel), + BlockSection.emptyArray + ).get + + history.readPopowProofBytesFromDb() shouldBe None + history.historyStorage + .getIndex(history.LegacyNipopowSnapshotKey) + .get + .toSeq shouldBe legacySentinel.toSeq + } finally { + history.closeStorage() + } + } + + property("V2 NiPoPoW cache bytes survive history restart") { + val directory = createTempDir + val baseSettings = ErgoNodeTestConstants.initSettings + val historySettings = baseSettings.copy( + directory = directory.getAbsolutePath, + nodeSettings = baseSettings.nodeSettings.copy(extraIndex = false) + ) + val cachedBytes = Array[Byte](10, 20, 30, 40) + val firstHistory = ErgoHistory.readOrGenerate(historySettings)(null) + try { + firstHistory.historyStorage.insert( + Array(firstHistory.NipopowProofV2Key -> cachedBytes), + BlockSection.emptyArray + ).get + } finally { + firstHistory.closeStorage() + } + + val reopenedHistory = ErgoHistory.readOrGenerate(historySettings)(null) + try { + reopenedHistory.readPopowProofBytesFromDb().get.toSeq shouldBe cachedBytes.toSeq + reopenedHistory.readPopowProofBytesFromDb().get.toSeq shouldBe cachedBytes.toSeq + } finally { + reopenedHistory.closeStorage() + } + } + + property("periodic NiPoPoW snapshots refresh V2 without rewriting legacy bytes") { + val history = genHistory(None, popowBootstrap = false) + try { + history.historyStorage.insert( + Array(history.LegacyNipopowSnapshotKey -> legacySentinel), + BlockSection.emptyArray + ).get + + val chain = blockStream(None) + .take(ErgoNodeTestConstants.settings.chainSettings.makeSnapshotEvery) + applyChain(history, chain) + + history.readPopowProofBytesFromDb().isDefined shouldBe true + history.historyStorage + .getIndex(history.LegacyNipopowSnapshotKey) + .get + .toSeq shouldBe legacySentinel.toSeq + } finally { + history.closeStorage() + } + } + property("popow proof application") { val senderHistory = genHistory(None, popowBootstrap = false) val senderChain = genChain(5000, senderHistory) @@ -32,4 +171,22 @@ class PopowProcessorSpecification extends ErgoCorePropertyTest { receiverHistory.bestHeaderOpt.get shouldBe senderHistory.bestHeaderOpt.get } + property("popow proof application rejects headers failing real Autolykos validation") { + val senderHistory = genHistory(None, popowBootstrap = false) + val senderChain = genChain(80, senderHistory) + applyChain(senderHistory, senderChain) + + val popowProofBytes = senderHistory.popowProofBytes().get + val realPowScheme = new AutolykosPowScheme(baseSettings.chainSettings.powScheme.k, baseSettings.chainSettings.powScheme.n) + val receiverHistory = genRealPowHistory(senderHistory.bestHeaderAtHeight(1).map(_.id), realPowScheme) + val popowProof = receiverHistory.nipopowSerializer.parseBytes(popowProofBytes) + + popowProof.headersChain.exists(h => realPowScheme.validate(h).isFailure) shouldBe true + + receiverHistory.headersHeight shouldBe 0 + receiverHistory.applyPopowProof(popowProof) + receiverHistory.headersHeight shouldBe 0 + receiverHistory.bestHeaderOpt shouldBe None + } + } diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala index 1520a9f032..d35e22703d 100644 --- a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala @@ -10,6 +10,7 @@ import org.ergoplatform.settings.{ErgoSettings, ErgoValidationSettingsUpdate, Pa import org.ergoplatform.utils.{ErgoTestHelpers, RandomWrapper} import org.scalatest.flatspec.AnyFlatSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks +import scorex.crypto.authds.ADKey import scorex.util.encode.Base16 import sigma.ast.ByteArrayConstant import sigma.Colls @@ -29,6 +30,13 @@ class ErgoMemPoolSpec extends AnyFlatSpec import org.ergoplatform.utils.generators.ErgoCoreTransactionGenerators._ import org.ergoplatform.utils.generators.ValidBlocksGenerators._ + private def feeTx(inputSeed: Byte, fee: Long): ErgoTransaction = { + ErgoTransaction( + IndexedSeq(new Input(ADKey @@ Array.fill(32)(inputSeed), emptyProverResult)), + IndexedSeq(new ErgoBoxCandidate(fee, feeProp, creationHeight = 0)) + ) + } + it should "accept valid transaction" in { val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) @@ -486,6 +494,31 @@ class ErgoMemPoolSpec extends AnyFlatSpec pool.stats.takenTxns shouldBe (family_depth + 1) * txs.size } + it should "not recommend fee below node minimal fee" in { + val feeSettings = settings.copy(nodeSettings = settings.nodeSettings.copy(minimalFeeAmount = 1000000L)) + val minimalFee = feeSettings.nodeSettings.minimalFeeAmount + val now = System.currentTimeMillis() + val lowFeeHistogram = FeeHistogramBin(nTxns = 1, totalFee = minimalFee / 2) :: + List.fill(MemPoolStatistics.nHistogramBins - 1)(FeeHistogramBin(0, 0)) + val stats = MemPoolStatistics(now, takenTxns = 1, snapTime = now, histogram = lowFeeHistogram) + val pool = new ErgoMemPool(OrderedTxPool.empty(feeSettings), stats, SortingOption.FeePerByte)(feeSettings) + + pool.getRecommendedFee(expectedWaitTimeMinutes = 0, txSize = 1024) shouldBe minimalFee + } + + it should "not let idle uptime dominate expected wait time" in { + val feeSettings = settings.copy(nodeSettings = settings.nodeSettings.copy(minimalFeeAmount = 1000000L)) + val minimalFee = feeSettings.nodeSettings.minimalFeeAmount + val poolWithHigherFeeTx = ErgoMemPool.empty(feeSettings) + .put(UnconfirmedTransaction(feeTx(inputSeed = 1, fee = minimalFee * 100), None)) + val now = System.currentTimeMillis() + val staleMeasurementStart = now - 365L * 24 * 60 * 60 * 1000 + val staleStats = MemPoolStatistics(staleMeasurementStart, takenTxns = 1, snapTime = now) + val pool = new ErgoMemPool(poolWithHigherFeeTx.pool, staleStats, SortingOption.FeePerByte)(feeSettings) + + pool.getExpectedWaitTime(txFee = minimalFee, txSize = 1024) should be <= MemPoolStatistics.measurementIntervalMsec.toLong + } + it should "put not adding transaction twice" in { val pool = ErgoMemPool.empty(settings).pool val tx = invalidErgoTransactionGen.sample.get diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala index cc261eebc5..284952b6f4 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala @@ -7,7 +7,7 @@ import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransacti import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader import org.ergoplatform.nodeView.wallet.WalletScanLogic.ScanResults import org.ergoplatform.nodeView.wallet.persistence.{OffChainRegistry, WalletRegistry, WalletStorage} -import org.ergoplatform.nodeView.wallet.requests.{AssetIssueRequest, PaymentRequest} +import org.ergoplatform.nodeView.wallet.requests.{AssetIssueRequest, BurnTokensRequest, PaymentRequest} import org.ergoplatform.nodeView.wallet.scanning.{EqualsScanningPredicate, ScanRequest, ScanWalletInteraction} import org.ergoplatform.sdk.SecretString import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, ExtendedSecretKey} @@ -28,6 +28,7 @@ import scorex.db.{LDBKVStore, LDBVersionedStore} import scorex.util.encode.Base16 import sigma.Extensions.ArrayOps import sigma.ast.{ByteArrayConstant, EvaluatedValue, FalseLeaf, SType} +import sigmastate.eval.Extensions._ import sigmastate.helpers.TestingHelpers.testBox import scala.collection.compat.immutable.ArraySeq @@ -274,6 +275,101 @@ class ErgoWalletServiceSpec } } + property("asset issuance should be independent of burn request order") { + withVersionedStore(2) { versionedStore => + withStore { store => + val wState = initialState(store, versionedStore) + val existingAssetAmount = 10L + val burnAmount = 3L + val issueAmount = 7L + val inputBoxes = boxesAvailable( + makeGenesisBlock(pks.head.pubkey, Seq(newAssetIdStub -> existingAssetAmount)), + pks.head.pubkey + ) + val existingTokenId = inputBoxes.flatMap(_.additionalTokens.toArray).head._1 + val encodedBoxes = inputBoxes.map(box => Base16.encode(ErgoBoxSerializer.toBytes(box))) + val burnRequest = BurnTokensRequest(Array(existingTokenId -> burnAmount)) + val paymentRequest = PaymentRequest(pks.head, 1000000L, Array.empty, Map.empty) + val issueRequest = AssetIssueRequest( + address = pks.head, + valueOpt = Some(10000000L), + amount = issueAmount, + name = "test-name", + description = "test-description", + decimals = 4, + registers = Option.empty + ) + val boxSelector = new ReplaceCompactCollectBoxSelector( + settings.walletSettings.maxInputs, + settings.walletSettings.optimalInputs, + None + ) + + val requestOrders = Seq( + Seq(burnRequest, issueRequest), + Seq(issueRequest, burnRequest) + ) ++ Seq(burnRequest, issueRequest, paymentRequest).permutations.toSeq + + requestOrders.foreach { requests => + val result = generateUnsignedTransaction( + wState, + boxSelector, + requests, + inputsRaw = encodedBoxes, + dataInputsRaw = Seq.empty + ) + val requestOrder = requests.map(_.getClass.getSimpleName).mkString(", ") + withClue(s"request order: $requestOrder; failure: ${result.failed.map(_.getMessage).toOption}") { + result.isSuccess shouldBe true + } + + val (tx, selectedInputs, _) = result.get + val issuedTokenId = selectedInputs.head.id.toTokenId + val issueOutputs = tx.outputCandidates.filter( + _.additionalTokens.toArray.exists { case (tokenId, _) => tokenId == issuedTokenId } + ) + issueOutputs should have size 1 + issueOutputs.head.value shouldBe issueRequest.valueOpt.get + issueOutputs.head.ergoTree shouldBe pks.head.script + issueOutputs.head.additionalTokens.toArray should contain(issuedTokenId -> issueAmount) + issueOutputs.head.additionalRegisters shouldBe Map( + ErgoBox.R4 -> ByteArrayConstant("test-name".getBytes("UTF-8")), + ErgoBox.R5 -> ByteArrayConstant("test-description".getBytes("UTF-8")), + ErgoBox.R6 -> ByteArrayConstant("4".getBytes("UTF-8")) + ) + + if (requests.contains(paymentRequest)) { + val paymentOutputs = tx.outputCandidates.filter(_.value == paymentRequest.value) + paymentOutputs should have size 1 + paymentOutputs.head.ergoTree shouldBe pks.head.script + paymentOutputs.head.additionalTokens.toArray shouldBe empty + paymentOutputs.head.additionalRegisters shouldBe empty + } + + selectedInputs + .flatMap(_.additionalTokens.toArray) + .collect { case (tokenId, amount) if tokenId == issuedTokenId => amount } + .sum shouldBe 0L + tx.outputCandidates + .flatMap(_.additionalTokens.toArray) + .collect { case (tokenId, amount) if tokenId == issuedTokenId => amount } + .sum shouldBe issueAmount + + val selectedExistingAmount = selectedInputs + .flatMap(_.additionalTokens.toArray) + .collect { case (tokenId, amount) if tokenId == existingTokenId => amount } + .sum + val outputExistingAmount = tx.outputCandidates + .flatMap(_.additionalTokens.toArray) + .collect { case (tokenId, amount) if tokenId == existingTokenId => amount } + .sum + selectedExistingAmount - outputExistingAmount shouldBe burnAmount + selectedInputs.map(_.value).sum shouldBe tx.outputCandidates.map(_.value).sum + } + } + } + } + property("it should process unlock using preEip3Derivation") { withVersionedStore(2) { versionedStore => withStore { store => diff --git a/src/test/scala/org/ergoplatform/serialization/SerializationTests.scala b/src/test/scala/org/ergoplatform/serialization/SerializationTests.scala index 8cb7de1e37..397a54e442 100644 --- a/src/test/scala/org/ergoplatform/serialization/SerializationTests.scala +++ b/src/test/scala/org/ergoplatform/serialization/SerializationTests.scala @@ -1,16 +1,91 @@ package org.ergoplatform.serialization -import org.ergoplatform.modifiers.history.popow.NipopowProofSerializer +import org.ergoplatform.modifiers.history.popow.{NipopowProof, NipopowProofSerializer, PoPowHeaderSerializer, PoPowParams} import org.ergoplatform.network.ErgoNodeViewSynchronizer import org.ergoplatform.nodeView.wallet.persistence.WalletDigestSerializer import org.ergoplatform.utils.ErgoCorePropertyTest import org.ergoplatform.utils.ErgoCoreTestConstants.nipopowAlgos -import org.ergoplatform.utils.generators.ErgoNodeGenerators.poPowProofGen +import org.ergoplatform.utils.generators.ErgoNodeGenerators.{poPowProofGen, validNiPoPowProofGen} +import scorex.util.serialization.VLQByteStringWriter class SerializationTests extends ErgoCorePropertyTest with org.ergoplatform.utils.SerializationTests { import org.ergoplatform.utils.generators.ErgoNodeWalletGenerators._ import org.ergoplatform.utils.generators.ErgoCoreTransactionGenerators._ + private val nipopowSerializer = new NipopowProofSerializer(nipopowAlgos) + + private sealed trait LengthPrefixedSite + private case object PrefixElement extends LengthPrefixedSite + private case object SuffixHeadElement extends LengthPrefixedSite + private case object SuffixTailElement extends LengthPrefixedSite + + private val MaxProofElements = PoPowParams.MaxProofElements + private val MaxHeaderBytes = PoPowHeaderSerializer.MaxHeaderBytes + private val MaxPoPowHeaderBytes = PoPowHeaderSerializer.MaxSerializedBytes + + private def smallValidProof(): NipopowProof = validNiPoPowProofGen(1, 1).sample.get + + private lazy val sampleProof: NipopowProof = { + val proof = validNiPoPowProofGen(1, 2).sample.get + require(proof.prefix.nonEmpty, "length-prefix fixture needs a prefix element") + require(proof.suffixTail.nonEmpty, "length-prefix fixture needs a suffix-tail element") + proof + } + + private def uintBytes(value: Long): Array[Byte] = + (new VLQByteStringWriter).putUInt(value).toBytes + + private def replaceUInt(bytes: Array[Byte], offset: Int, original: Int, replacement: Long): Array[Byte] = { + val originalLength = uintBytes(original.toLong).length + bytes.take(offset) ++ uintBytes(replacement) ++ bytes.drop(offset + originalLength) + } + + private def writerBytes(write: VLQByteStringWriter => Unit): Array[Byte] = { + val writer = new VLQByteStringWriter + write(writer) + writer.toBytes + } + + private def putLengthPrefixed(writer: VLQByteStringWriter, + bytes: Array[Byte], + mutate: Boolean, + declaredDelta: Int, + fillerLength: Int): Unit = { + val declaredSize = bytes.length + (if (mutate) declaredDelta else 0) + require(declaredSize >= 0) + writer.putUInt(declaredSize.toLong) + writer.putBytes(bytes) + if (mutate && fillerLength > 0) { + writer.putBytes(Array.fill(fillerLength)(0x7f.toByte)) + } + } + + private def serializeWithDeclaredLengthMutation(proof: NipopowProof, + site: LengthPrefixedSite, + declaredDelta: Int, + fillerLength: Int): Array[Byte] = writerBytes { writer => + writer.putUInt(proof.m.toLong) + writer.putUInt(proof.k.toLong) + writer.putUInt(proof.prefix.length.toLong) + proof.prefix.zipWithIndex.foreach { case (header, index) => + putLengthPrefixed( + writer, header.bytes, site == PrefixElement && index == 0, declaredDelta, fillerLength) + } + putLengthPrefixed( + writer, proof.suffixHead.bytes, site == SuffixHeadElement, declaredDelta, fillerLength) + writer.putUInt(proof.suffixTail.length.toLong) + proof.suffixTail.zipWithIndex.foreach { case (header, index) => + putLengthPrefixed( + writer, header.bytes, site == SuffixTailElement && index == 0, declaredDelta, fillerLength) + } + writer.put(if (proof.continuous) 1 else 0) + } + + private def assertProofParseFailureContains(bytes: Array[Byte], expected: String): Unit = { + val failure = nipopowSerializer.parseBytesTry(bytes).failed.get + failure.toString should include(expected) + } + property("Serializers should be defined for all block sections") { val block = invalidErgoFullBlockGen.sample.get block.toSeq.foreach { s => @@ -25,7 +100,118 @@ class SerializationTests extends ErgoCorePropertyTest with org.ergoplatform.util } property("PoPowProof serialization") { - checkSerializationRoundtrip(poPowProofGen, new NipopowProofSerializer(nipopowAlgos)) + checkSerializationRoundtrip(poPowProofGen, nipopowSerializer) + } + + property("PoPowProof parser rejects invalid m") { + val proof = smallValidProof() + val bytes = nipopowSerializer.toBytes(proof) + Seq(0L, 20001L).foreach { invalidM => + withClue(s"m=$invalidM") { + val mutated = replaceUInt(bytes, offset = 0, proof.m, invalidM) + nipopowSerializer.parseBytesTry(mutated) shouldBe 'failure + } + } + } + + property("PoPowProof parser rejects invalid k") { + val proof = smallValidProof() + val bytes = nipopowSerializer.toBytes(proof) + val kOffset = uintBytes(proof.m.toLong).length + Seq(0L, 20001L).foreach { invalidK => + withClue(s"k=$invalidK") { + val mutated = replaceUInt(bytes, kOffset, proof.k, invalidK) + nipopowSerializer.parseBytesTry(mutated) shouldBe 'failure + } + } + } + + property("PoPowProof parser rejects suffix length different from k") { + val proof = smallValidProof() + val bytes = nipopowSerializer.toBytes(proof) + val kOffset = uintBytes(proof.m.toLong).length + val mutated = replaceUInt(bytes, kOffset, proof.k, proof.k + 1L) + + nipopowSerializer.parseBytesTry(mutated) shouldBe 'failure + } + + property("PoPowProof parser rejects an invalid continuous mode byte") { + val bytes = nipopowSerializer.toBytes(smallValidProof()) + bytes(bytes.length - 1) = 2 + + assertProofParseFailureContains(bytes, "continuous mode") + } + + property("PoPowProof declared element lengths define outer slicing") { + Seq(PrefixElement, SuffixHeadElement, SuffixTailElement).foreach { site => + withClue(s"site=$site canonical") { + nipopowSerializer.parseBytes( + serializeWithDeclaredLengthMutation(sampleProof, site, 0, 0)) shouldBe sampleProof + } + withClue(s"site=$site under-declared") { + nipopowSerializer.parseBytesTry( + serializeWithDeclaredLengthMutation(sampleProof, site, -1, 0)) shouldBe 'failure + } + withClue(s"site=$site over-declared without filler") { + nipopowSerializer.parseBytesTry( + serializeWithDeclaredLengthMutation(sampleProof, site, 1, 0)) shouldBe 'failure + } + withClue(s"site=$site over-declared with matching filler") { + nipopowSerializer.parseBytes( + serializeWithDeclaredLengthMutation(sampleProof, site, 1, 1)) shouldBe sampleProof + } + } + } + + property("PoPowProof rejects an oversized prefix count before iterating") { + val bytes = writerBytes { writer => + writer.putUInt(1) + writer.putUInt(1) + writer.putUInt(MaxProofElements + 1L) + } + + assertProofParseFailureContains(bytes, "prefix count") + } + + property("PoPowProof rejects an oversized suffix count before iterating") { + val proof = smallValidProof() + val bytes = writerBytes { writer => + writer.putUInt(proof.m.toLong) + writer.putUInt(proof.k.toLong) + writer.putUInt(proof.prefix.length.toLong) + proof.prefix.foreach(header => putLengthPrefixed(writer, header.bytes, false, 0, 0)) + putLengthPrefixed(writer, proof.suffixHead.bytes, false, 0, 0) + writer.putUInt(MaxProofElements + 1L) + } + + assertProofParseFailureContains(bytes, "suffix count") + } + + property("PoPowProof rejects oversized length-prefixed elements before reading payloads") { + val prefixBytes = writerBytes { writer => + writer.putUInt(1) + writer.putUInt(1) + writer.putUInt(1) + writer.putUInt(MaxPoPowHeaderBytes + 1L) + } + val suffixHeadBytes = writerBytes { writer => + writer.putUInt(1) + writer.putUInt(1) + writer.putUInt(0) + writer.putUInt(MaxPoPowHeaderBytes + 1L) + } + val suffixTailBytes = writerBytes { writer => + writer.putUInt(1) + writer.putUInt(2) + writer.putUInt(0) + putLengthPrefixed(writer, sampleProof.suffixHead.bytes, false, 0, 0) + writer.putUInt(1) + writer.putUInt(MaxHeaderBytes + 1L) + } + + assertProofParseFailureContains(prefixBytes, "prefix element length") + assertProofParseFailureContains(suffixHeadBytes, "suffix head length") + assertProofParseFailureContains(suffixTailBytes, "suffix tail length") } } diff --git a/src/test/scala/org/ergoplatform/utils/generators/ErgoNodeGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ErgoNodeGenerators.scala index b4043fe3e7..c4b03e7072 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ErgoNodeGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ErgoNodeGenerators.scala @@ -24,7 +24,7 @@ object ErgoNodeGenerators { } yield { val chain = genHeaderChain(m * mulM + k, diffBitsOpt = None, useRealTs = false) val popowChain = popowHeaderChain(chain) - val params = PoPowParams(m, k, continuous = false) + val params = PoPowParams(m, k, continuous = false).get nipopowAlgos.prove(popowChain)(params).get } } diff --git a/src/test/scala/scorex/core/network/NetworkControllerSpec.scala b/src/test/scala/scorex/core/network/NetworkControllerSpec.scala index 8bc287d6d9..1265b535ab 100644 --- a/src/test/scala/scorex/core/network/NetworkControllerSpec.scala +++ b/src/test/scala/scorex/core/network/NetworkControllerSpec.scala @@ -3,6 +3,7 @@ package scorex.core.network import akka.actor.ActorRef import akka.io.Tcp import akka.testkit.{TestActorRef, TestProbe} +import org.ergoplatform.network.ErgoNodeViewSynchronizerMessages.DisconnectedPeer import org.ergoplatform.network.message.MessageConstants.MessageCode import org.ergoplatform.network.peer.PeerInfo import org.ergoplatform.utils.ErgoCorePropertyTest @@ -25,6 +26,8 @@ class NetworkControllerSpec extends ErgoCorePropertyTest { val scorexContext: ScorexContext = ScorexContext(Seq.empty, None, None) + case class EstablishedConnection(connectionProbe: TestProbe, handlerRef: ActorRef) + def createController(maxConnections: Int): (TestActorRef[NetworkController], TestProbe, TestProbe) = { val peerManagerProbe = TestProbe("PeerManager") val tcpManagerProbe = TestProbe("TcpManager") @@ -56,6 +59,33 @@ class NetworkControllerSpec extends ErgoCorePropertyTest { peerManagerProbe: TestProbe, remoteAddress: InetSocketAddress ): InetSocketAddress = { + beginIncomingConnection(controller, peerManagerProbe, remoteAddress) + remoteAddress + } + + def establishIncomingConnectionWithHandler( + controller: TestActorRef[NetworkController], + peerManagerProbe: TestProbe, + remoteAddress: InetSocketAddress + ): EstablishedConnection = { + val connectionProbe = beginIncomingConnection( + controller, + peerManagerProbe, + remoteAddress + ) + + val handlerRef = connectionProbe.expectMsgType[Tcp.Register].handler + connectionProbe.expectMsg(Tcp.ResumeReading) + connectionProbe.expectMsgType[Tcp.Write] + + EstablishedConnection(connectionProbe, handlerRef) + } + + private def beginIncomingConnection( + controller: TestActorRef[NetworkController], + peerManagerProbe: TestProbe, + remoteAddress: InetSocketAddress + ): TestProbe = { val localAddress = settings.scorexSettings.network.bindAddress val connectionProbe = TestProbe("Connection") @@ -66,7 +96,7 @@ class NetworkControllerSpec extends ErgoCorePropertyTest { controller ! ConnectionConfirmed(ConnectionId(remoteAddress, localAddress, Incoming), handlerRef) } - remoteAddress + connectionProbe } def establishOutgoingConnection( @@ -191,6 +221,97 @@ class NetworkControllerSpec extends ErgoCorePropertyTest { } } + property("blacklisting should close exactly the live connections for the banned IP") { + withFixture { f => + implicit val system = f.system + val (controller, peerManagerProbe, _) = f.createController(maxConnections = 30) + val disconnectProbe = TestProbe("DisconnectedPeers") + f.system.eventStream.subscribe(disconnectProbe.ref, classOf[DisconnectedPeer]) + + val firstAddress = new InetSocketAddress("192.0.2.10", 9101) + val secondAddress = new InetSocketAddress("192.0.2.10", 9102) + val unrelatedAddress = new InetSocketAddress("198.51.100.20", 9201) + val first = f.establishIncomingConnectionWithHandler( + controller, + peerManagerProbe, + firstAddress + ) + val second = f.establishIncomingConnectionWithHandler( + controller, + peerManagerProbe, + secondAddress + ) + val unrelated = f.establishIncomingConnectionWithHandler( + controller, + peerManagerProbe, + unrelatedAddress + ) + + peerManagerProbe.send(controller, Blacklisted(firstAddress)) + + first.connectionProbe.expectMsg(Tcp.Abort) + second.connectionProbe.expectMsg(Tcp.Abort) + unrelated.connectionProbe.expectNoMessage(200.millis) + + val duplicateBeforeTermination = TestProbe("DuplicateBeforeTermination") + duplicateBeforeTermination.send( + controller, + Tcp.Connected(secondAddress, settings.scorexSettings.network.bindAddress) + ) + duplicateBeforeTermination.expectMsg(Tcp.Close) + + first.connectionProbe.watch(first.handlerRef) + second.connectionProbe.watch(second.handlerRef) + first.connectionProbe.send(first.handlerRef, Tcp.Aborted) + second.connectionProbe.send(second.handlerRef, Tcp.Aborted) + first.connectionProbe.expectTerminated(first.handlerRef) + second.connectionProbe.expectTerminated(second.handlerRef) + + val disconnectedAddresses = disconnectProbe.receiveN(2, 2.seconds).collect { + case DisconnectedPeer(peer) => peer.connectionId.remoteAddress + }.toSet + disconnectedAddresses shouldBe Set(firstAddress, secondAddress) + + val replacement = TestProbe("ReplacementConnection") + replacement.send( + controller, + Tcp.Connected(secondAddress, settings.scorexSettings.network.bindAddress) + ) + peerManagerProbe.expectMsgPF(1.second) { + case ConfirmConnection(connectionId, connectionRef) => + connectionId.remoteAddress shouldBe secondAddress + connectionRef shouldBe replacement.ref + } + + val unrelatedDuplicate = TestProbe("UnrelatedDuplicate") + unrelatedDuplicate.send( + controller, + Tcp.Connected(unrelatedAddress, settings.scorexSettings.network.bindAddress) + ) + unrelatedDuplicate.expectMsg(Tcp.Close) + } + } + + property("blacklisting should match by IP when the exact socket is absent") { + withFixture { f => + val (controller, peerManagerProbe, _) = f.createController(maxConnections = 30) + val siblingAddress = new InetSocketAddress("192.0.2.30", 9301) + val missingSocketAddress = new InetSocketAddress("192.0.2.30", 9399) + val sibling = f.establishIncomingConnectionWithHandler( + controller, + peerManagerProbe, + siblingAddress + ) + + peerManagerProbe.send(controller, Blacklisted(missingSocketAddress)) + + sibling.connectionProbe.expectMsg(Tcp.Abort) + sibling.connectionProbe.watch(sibling.handlerRef) + sibling.connectionProbe.send(sibling.handlerRef, Tcp.Aborted) + sibling.connectionProbe.expectTerminated(sibling.handlerRef) + } + } + property("outgoing connection should be accepted when total below maxConnections") { withFixture { f => val (controller, peerManagerProbe, tcpManagerProbe) = f.createController(maxConnections = 10) diff --git a/src/test/scala/scorex/core/network/PeerConnectionHandlerSpecification.scala b/src/test/scala/scorex/core/network/PeerConnectionHandlerSpecification.scala new file mode 100644 index 0000000000..f8319787f4 --- /dev/null +++ b/src/test/scala/scorex/core/network/PeerConnectionHandlerSpecification.scala @@ -0,0 +1,167 @@ +package scorex.core.network + +import akka.io.Tcp +import akka.testkit.{TestActorRef, TestProbe} +import akka.util.ByteString +import org.ergoplatform.network.message.{ + GetPeersSpec, + Message, + MessageSpec, + UtxoSnapshotChunkSpec +} +import org.ergoplatform.network.{Handshake, HandshakeSerializer} +import org.ergoplatform.utils.ErgoCorePropertyTest +import org.ergoplatform.utils.ErgoNodeTestConstants.{defaultPeerSpec, settings} +import scorex.core.app.ScorexContext +import scorex.testkit.utils.AkkaFixture + +import java.net.InetSocketAddress +import scala.concurrent.Await +import scala.concurrent.duration.{Duration, DurationInt} + +class PeerConnectionHandlerSpecification extends ErgoCorePropertyTest { + private final class ConnectedHandler(val connection: TestProbe, + val watcher: TestProbe, + val handler: TestActorRef[PeerConnectionHandler]) + + private def withConnectedHandler( + messageSpecs: Seq[MessageSpec[_]], + localPort: Int + )(test: ConnectedHandler => Unit): Unit = { + val fixture = new AkkaFixture + try { + implicit val system = fixture.system + implicit val ec = system.dispatcher + val connection = TestProbe("connection") + val controller = TestProbe("controller") + val localAddress = new InetSocketAddress("127.0.0.1", localPort) + val remoteAddress = new InetSocketAddress("127.0.0.1", localPort + 1) + val description = ConnectionDescription( + connection.ref, + ConnectionId(remoteAddress, localAddress, Incoming), + Some(localAddress), + Seq.empty + ) + val handler = TestActorRef(new PeerConnectionHandler( + settings.scorexSettings, + controller.ref, + ScorexContext(messageSpecs, None, None), + description + )) + + connection.expectMsgType[Tcp.Register] + connection.expectMsg(Tcp.ResumeReading) + connection.expectMsgType[Tcp.Write] + + val handshake = HandshakeSerializer.toBytes( + Handshake(defaultPeerSpec, System.currentTimeMillis()) + ) + connection.send(handler, Tcp.Received(ByteString(handshake))) + controller.expectMsgType[NetworkController.ReceivableMessages.Handshaked] + connection.expectMsg(Tcp.ResumeReading) + controller.watch(handler) + + test(new ConnectedHandler(connection, controller, handler)) + } finally { + Await.result(fixture.system.terminate(), Duration.Inf) + } + } + + property("abort before a fifth maximum snapshot frame is retained") { + withConnectedHandler(Seq(UtxoSnapshotChunkSpec), localPort = 9083) { fixture => + val chunkMessage = Message( + UtxoSnapshotChunkSpec, + Right(Array.fill[Byte](3999996)(1)), + None + ) + fixture.handler ! chunkMessage + val failedWrite = fixture.connection.expectMsgType[Tcp.Write] + failedWrite.data.length shouldEqual 4000013 + fixture.connection.send(fixture.handler, Tcp.CommandFailed(failedWrite)) + fixture.connection.expectMsg(Tcp.ResumeWriting) + + (2 to 4).foreach { id => + val write = Tcp.Write( + failedWrite.data, + PeerConnectionHandler.ReceivableMessages.Ack(id) + ) + fixture.connection.send(fixture.handler, Tcp.CommandFailed(write)) + fixture.connection.expectMsg(Tcp.ResumeWriting) + } + fixture.connection.expectNoMessage(200.millis) + + val overLimitWrite = Tcp.Write( + failedWrite.data, + PeerConnectionHandler.ReceivableMessages.Ack(5) + ) + fixture.connection.send( + fixture.handler, + Tcp.CommandFailed(overLimitWrite) + ) + fixture.connection.expectMsg(Tcp.ResumeWriting) + fixture.connection.expectMsg(1.second, Tcp.Abort) + fixture.watcher.expectTerminated(fixture.handler) + } + } + + property("abort before more than 64 outbound messages are buffered") { + withConnectedHandler(Seq(GetPeersSpec), localPort = 9093) { fixture => + val getPeersMessage = Message(GetPeersSpec, Right(()), None) + fixture.handler ! getPeersMessage + val failedWrite = fixture.connection.expectMsgType[Tcp.Write] + fixture.connection.send(fixture.handler, Tcp.CommandFailed(failedWrite)) + fixture.connection.expectMsg(Tcp.ResumeWriting) + + (1 until PeerConnectionHandler.MaxBufferedOutboundMessages) + .foreach(_ => fixture.handler ! getPeersMessage) + fixture.connection.expectNoMessage(200.millis) + + fixture.handler ! getPeersMessage + fixture.connection.expectMsg(1.second, Tcp.Abort) + fixture.watcher.expectTerminated(fixture.handler) + } + } + + property("account retried and acknowledged writes exactly") { + withConnectedHandler( + Seq(UtxoSnapshotChunkSpec), + localPort = 9103 + ) { fixture => + val chunkMessage = Message( + UtxoSnapshotChunkSpec, + Right(Array.fill[Byte](3999996)(1)), + None + ) + fixture.handler ! chunkMessage + val failedWrite = fixture.connection.expectMsgType[Tcp.Write] + failedWrite.data.length shouldEqual 4000013 + failedWrite.ack shouldEqual PeerConnectionHandler.ReceivableMessages.Ack(1) + + fixture.connection.send(fixture.handler, Tcp.CommandFailed(failedWrite)) + fixture.connection.expectMsg(Tcp.ResumeWriting) + + (2 to 4).foreach(_ => fixture.handler ! chunkMessage) + fixture.connection.expectNoMessage(200.millis) + + fixture.connection.send(fixture.handler, Tcp.WritingResumed) + val retriedWrite = fixture.connection.expectMsgType[Tcp.Write] + retriedWrite.data shouldEqual failedWrite.data + retriedWrite.ack shouldEqual PeerConnectionHandler.ReceivableMessages.Ack(1) + fixture.connection.send(fixture.handler, Tcp.CommandFailed(retriedWrite)) + fixture.connection.expectMsg(Tcp.ResumeWriting) + fixture.connection.expectNoMessage(200.millis) + + fixture.connection.send(fixture.handler, Tcp.WritingResumed) + val finalRetry = fixture.connection.expectMsgType[Tcp.Write] + finalRetry.data shouldEqual failedWrite.data + finalRetry.ack shouldEqual PeerConnectionHandler.ReceivableMessages.Ack(1) + fixture.connection.send( + fixture.handler, + PeerConnectionHandler.ReceivableMessages.Ack(1) + ) + + val nextWrite = fixture.connection.expectMsgType[Tcp.Write] + nextWrite.ack shouldEqual PeerConnectionHandler.ReceivableMessages.Ack(2) + } + } +}