Merge pull request #1810 from ergoplatform/v4.0.41

Candidate for 4.0.41

README.md

@@ -82,7 +82,7 @@ To run specific Ergo version `<VERSION>` as a service with custom config `/path/
         -e MAX_HEAP=3G \
         ergoplatform/ergo:<VERSION> --<networkId> -c /etc/myergo.conf
 
-Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.40`.
+Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.41`.
 
 This will connect to the Ergo mainnet or testnet following your configuration passed in `myergo.conf` and network flag `--<networkId>`. Every default config value would be overwritten with corresponding value in `myergo.conf`. `MAX_HEAP` variable can be used to control how much memory can the node consume.
 

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala

@@ -8,6 +8,8 @@ import scorex.crypto.hash
 import scorex.crypto.hash.{CryptographicHash, Digest}
 import scorex.db.LDBVersionedStore
 import scorex.util.ScorexLogging
+
+import scala.collection.mutable
 import scala.util.{Failure, Try}
 
 /**
@@ -63,7 +65,7 @@ class VersionedLDBAVLStorage[D <: Digest](store: LDBVersionedStore,
   }
 
   private def serializedVisitedNodes(node: ProverNodes[D],
-                                     isTop: Boolean): Seq[(Array[Byte], Array[Byte])] = {
+                                     isTop: Boolean): Array[(Array[Byte], Array[Byte])] = {
     // Should always serialize top node. It may not be new if it is the creation of the tree
     if (node.isNew || isTop) {
       val pair: (Array[Byte], Array[Byte]) = (nodeKey(node), toBytes(node))
@@ -72,28 +74,33 @@ class VersionedLDBAVLStorage[D <: Digest](store: LDBVersionedStore,
           val leftSubtree = serializedVisitedNodes(n.left, isTop = false)
           val rightSubtree = serializedVisitedNodes(n.right, isTop = false)
           pair +: (leftSubtree ++ rightSubtree)
-        case _: ProverLeaf[D] => Seq(pair)
+        case _: ProverLeaf[D] => Array(pair)
       }
     } else {
-      Seq.empty
+      Array.empty
     }
   }
 
   //TODO label or key???
   private def nodeKey(node: ProverNodes[D]): Array[Byte] = node.label
 
-  private def toBytes(node: ProverNodes[D]): Array[Byte] = node match {
-    case n: InternalProverNode[D] => InternalNodePrefix +: n.balance +: (n.key ++ n.left.label ++ n.right.label)
-    case n: ProverLeaf[D] =>
-      if (fixedSizeValueMode){
-        LeafPrefix +: (n.key ++ n.value ++ n.nextLeafKey)
-      } else {
-        LeafPrefix +: (n.key ++ Ints.toByteArray(n.value.length) ++ n.value ++ n.nextLeafKey)
-      }
+  private def toBytes(node: ProverNodes[D]): Array[Byte] = {
+    val builder = new mutable.ArrayBuilder.ofByte;
+    node match {
+      case n: InternalProverNode[D] =>
+        builder += InternalNodePrefix += n.balance ++= n.key ++= n.left.label ++= n.right.label
+      case n: ProverLeaf[D] =>
+        if (fixedSizeValueMode) {
+          builder += LeafPrefix ++= n.key ++= n.value ++= n.nextLeafKey
+        } else {
+          builder += LeafPrefix ++= n.key ++= Ints.toByteArray(n.value.length) ++= n.value ++= n.nextLeafKey
+        }
+    }
+    builder.result()
   }
 
   //todo: this method is not used, should be removed on next scrypto update?
-  override def update(prover: BatchAVLProver[D, _]): Try[Unit] = update(prover, Seq())
+  override def update(prover: BatchAVLProver[D, _]): Try[Unit] = update(prover, Nil)
 }
 
 

benchmarks/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolBenchmark.scala

@@ -1,6 +1,6 @@
 package org.ergoplatform.nodeView.mempool
 
-import org.ergoplatform.modifiers.mempool.ErgoTransaction
+import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction}
 import org.ergoplatform.utils.generators.ErgoTransactionGenerators
 import org.scalameter.KeyValue
 import org.scalameter.api._
@@ -43,7 +43,7 @@ object ErgoMemPoolBenchmark
 
   private def bench(txsInIncomeOrder: Seq[ErgoTransaction]): Unit = {
     var pool = ErgoMemPool.empty(settings)
-    txsInIncomeOrder.foreach(tx => pool = pool.put(tx).get)
+    txsInIncomeOrder.foreach(tx => pool = pool.put(UnconfirmedTransaction(tx)).get)
   }
 
   performance of "ErgoMemPool awaiting" in {

benchmarks/src/test/scala/org/ergoplatform/nodeView/mempool/MempoolPerformanceBench.scala

@@ -1,7 +1,7 @@
 package org.ergoplatform.nodeView.mempool
 
-import org.ergoplatform.modifiers.mempool.ErgoTransaction
-import org.ergoplatform.utils.generators.{ErgoTransactionGenerators, ErgoGenerators}
+import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction}
+import org.ergoplatform.utils.generators.{ErgoGenerators, ErgoTransactionGenerators}
 import org.scalacheck.Gen
 import org.scalatest.propspec.AnyPropSpec
 import scorex.testkit.properties.mempool.MempoolFilterPerformanceTest
@@ -14,4 +14,5 @@ class MempoolPerformanceBench extends AnyPropSpec
   override val memPool: ErgoMemPool = ErgoMemPool.empty(settings)
   override val memPoolGenerator: Gen[ErgoMemPool] = emptyMemPoolGen
   override val transactionGenerator: Gen[ErgoTransaction] = invalidErgoTransactionGen
+  override val unconfirmedTxGenerator: Gen[UnconfirmedTransaction] = invalidErgoTransactionGen.map(UnconfirmedTransaction.apply)
 }

src/main/resources/api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.2"
 
 info:
-  version: "4.0.40"
+  version: "4.0.41"
   title: Ergo Node API
   description: API docs for Ergo Node. Models are shared between all Ergo products
   contact:

src/main/resources/application.conf

@@ -61,8 +61,12 @@ ergo {
     # Maximum number of unconfirmed transactions node can accept
     mempoolCapacity = 1000
 
-    # Time window within which a node performs mempool cleanup in between blocks application
-    mempoolCleanupDuration = 20s
+    # Interval for mempool transaction re-check. We check transaction when it is entering the mempool, and then
+    # re-check it every interval valie
+    mempoolCleanupDuration = 30m
+
+    # Mempool transaction sorting scheme ("random", "bySize", or "byExecutionCost")
+    mempoolSorting = "random"
 
     # Number of transactions from mempool to be re-broadcasted at each epoch
     rebroadcastCount = 3
@@ -396,7 +400,7 @@ scorex {
     nodeName = "ergo-node"
 
     # Network protocol version to be sent in handshakes
-    appVersion = 4.0.40
+    appVersion = 4.0.41
 
     # Network agent name. May contain information about client code
     # stack, starting from core code-base up to the end graphical interface.

src/main/resources/mainnet.conf

@@ -49,8 +49,8 @@ ergo {
     # The node still applying transactions to UTXO set and so checks UTXO set digests for each block.
     # Block at checkpoint height is to be checked against expected one.
     checkpoint = {
-      height = 818258
-      blockId = "77f500f93890f7df6cc9d5214c0b1db6e9251c96915c716fb1aab6360d768e63"
+      height = 823877
+      blockId = "cb5242533e727bf73989ef3b286d031abf13b72b3145a6e06fa728eb0f7cb658"
     }
 
     # List with hex-encoded identifiers of transactions banned from getting into memory pool
@@ -65,7 +65,7 @@ scorex {
   network {
     magicBytes = [1, 0, 2, 4]
     bindAddress = "0.0.0.0:9030"
-    nodeName = "ergo-mainnet-4.0.40"
+    nodeName = "ergo-mainnet-4.0.41"
     nodeName = ${?NODENAME}
     knownPeers = [
       "213.239.193.208:9030",

src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala

@@ -8,7 +8,7 @@ import org.ergoplatform.{ErgoBox, ErgoLikeContext, ErgoLikeTransaction, JsonCode
 import org.ergoplatform.http.api.ApiEncoderOption.Detalization
 import org.ergoplatform.ErgoBox.RegisterId
 import org.ergoplatform.mining.{groupElemFromBytes, groupElemToBytes}
-import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction}
+import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction, UnsignedErgoTransaction}
 import org.ergoplatform.nodeView.history.ErgoHistory.Difficulty
 import org.ergoplatform.settings.ErgoAlgos
 import org.ergoplatform.nodeView.wallet.persistence.WalletDigest
@@ -195,6 +195,27 @@ trait ApiCodecs extends JsonCodecs {
     } yield ErgoTransaction(ergoLikeTx)
   }
 
+  // We are not using this encoder for now, but may use in future
+  implicit val unconfirmedTxEncoder: Encoder[UnconfirmedTransaction] = { unconfirmedTx =>
+    Json.obj(
+      "transaction" -> transactionEncoder(unconfirmedTx.transaction),
+      "lastCost" -> unconfirmedTx.lastCost.asJson,
+      "createdTime" -> unconfirmedTx.createdTime.asJson,
+      "lastCheckedTime" -> unconfirmedTx.lastCheckedTime.asJson
+    )
+  }
+
+  // We are not using this decoder for now, but may use in future
+  implicit val unconfirmedTxDecoder: Decoder[UnconfirmedTransaction] = { cursor =>
+    for {
+      tx <- transactionDecoder(cursor)
+      lastCost <- cursor.downField("lastCost").as[Option[Int]]
+      createdTime <- cursor.downField("createdTime").as[Long]
+      lastCheckedTime <- cursor.downField("lastCheckedTime").as[Long]
+    } yield UnconfirmedTransaction(tx, lastCost, createdTime, lastCheckedTime, Some(tx.bytes))
+  }
+
+
 
   implicit val sigmaBooleanEncoder: Encoder[SigmaBoolean] = {
     sigma =>

src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala

@@ -1,8 +1,9 @@
 package org.ergoplatform.http.api
 
 import akka.actor.ActorRef
-import akka.http.scaladsl.server.{Directive1, Route, ValidationRejection}
-import org.ergoplatform.modifiers.mempool.ErgoTransaction
+import akka.http.scaladsl.server.Route
+import akka.http.scaladsl.server.{Directive1, ValidationRejection}
+import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction}
 import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers}
 import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader
 import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoStateReader}
@@ -43,12 +44,12 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs {
     * Send local transaction to ErgoNodeViewHolder
     * @return Transaction Id with status OK(200), or BadRequest(400)
     */
-  protected def sendLocalTransactionRoute(nodeViewActorRef: ActorRef, tx: ErgoTransaction): Route = {
+  protected def sendLocalTransactionRoute(nodeViewActorRef: ActorRef, unconfirmedTx: UnconfirmedTransaction): Route = {
     val resultFuture =
-      (nodeViewActorRef ? LocallyGeneratedTransaction(tx))
+      (nodeViewActorRef ? LocallyGeneratedTransaction(unconfirmedTx))
         .mapTo[ProcessingOutcome]
         .flatMap {
-          case Accepted => Future.successful(tx.id)
+          case Accepted => Future.successful(unconfirmedTx.transaction.id)
           case DoubleSpendingLoser(_) => Future.failed(new IllegalArgumentException("Double spending attempt"))
           case Declined(ex) => Future.failed(ex)
           case Invalidated(ex) => Future.failed(ex)
@@ -67,14 +68,19 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs {
     */
   protected def verifyTransaction(tx: ErgoTransaction,
                                   readersHolder: ActorRef,
-                                  ergoSettings: ErgoSettings): Future[Try[ErgoTransaction]] = {
+                                  ergoSettings: ErgoSettings): Future[Try[UnconfirmedTransaction]] = {
+    val now: Long = System.currentTimeMillis()
+    val bytes = Some(tx.bytes)
+
     getStateAndPool(readersHolder)
       .map {
         case (utxo: UtxoStateReader, mp: ErgoMemPoolReader) =>
           val maxTxCost = ergoSettings.nodeSettings.maxTransactionCost
-          utxo.withMempool(mp).validateWithCost(tx, maxTxCost).map(_ => tx)
+          utxo.withMempool(mp)
+            .validateWithCost(tx, maxTxCost)
+            .map(cost => UnconfirmedTransaction(tx, Some(cost), now, now, bytes))
         case _ =>
-          tx.statelessValidity().map(_ => tx)
+          tx.statelessValidity().map(_ => UnconfirmedTransaction(tx, None, now, now, bytes))
       }
   }
 

src/main/scala/org/ergoplatform/http/api/TransactionsApiRoute.scala

@@ -5,7 +5,7 @@ import akka.http.scaladsl.server.{Directive, Route}
 import akka.pattern.ask
 import io.circe.Json
 import io.circe.syntax._
-import org.ergoplatform.modifiers.mempool.ErgoTransaction
+import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction}
 import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers}
 import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader
 import org.ergoplatform.nodeView.mempool.HistogramStats.getFeeHistogram
@@ -37,10 +37,11 @@ case class TransactionsApiRoute(readersHolder: ActorRef,
   private def getMemPool: Future[ErgoMemPoolReader] = (readersHolder ? GetReaders).mapTo[Readers].map(_.m)
 
   private def getUnconfirmedTransactions(offset: Int, limit: Int): Future[Json] = getMemPool.map { p =>
-    p.getAll.slice(offset, offset + limit).map(_.asJson).asJson
+    p.getAll.slice(offset, offset + limit).map(_.transaction.asJson).asJson
   }
 
-  private def validateTransactionAndProcess(tx: ErgoTransaction)(processFn: ErgoTransaction => Route): Route = {
+  private def validateTransactionAndProcess(tx: ErgoTransaction)
+                                           (processFn: UnconfirmedTransaction => Route): Route = {
     if (tx.size > ergoSettings.nodeSettings.maxTransactionSize) {
       BadRequest(s"Transaction $tx has too large size ${tx.size}")
     } else {
@@ -49,19 +50,19 @@ case class TransactionsApiRoute(readersHolder: ActorRef,
       } {
         _.fold(
           e => BadRequest(s"Malformed transaction: ${e.getMessage}"),
-          _ => processFn(tx)
+          utx => processFn(utx)
         )
       }
     }
   }
 
 
   def sendTransactionR: Route = (pathEnd & post & entity(as[ErgoTransaction])) { tx =>
-      validateTransactionAndProcess(tx)(validTx => sendLocalTransactionRoute(nodeViewActorRef, validTx))
-    }
+    validateTransactionAndProcess(tx)(validTx => sendLocalTransactionRoute(nodeViewActorRef, validTx))
+  }
 
   def checkTransactionR: Route = (path("check") & post & entity(as[ErgoTransaction])) { tx =>
-    validateTransactionAndProcess(tx)(validTx => ApiResponse(validTx.id))
+    validateTransactionAndProcess(tx)(validTx => ApiResponse(validTx.transaction.id))
   }
 
   def getUnconfirmedTransactionsR: Route = (path("unconfirmed") & get & txPaging) { (offset, limit) =>

src/main/scala/org/ergoplatform/http/api/WalletApiRoute.scala

@@ -7,7 +7,7 @@ import io.circe.syntax._
 import io.circe.{Encoder, Json}
 import org.ergoplatform._
 import org.ergoplatform.http.api.requests.HintExtractionRequest
-import org.ergoplatform.modifiers.mempool.ErgoTransaction
+import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction}
 import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers}
 import org.ergoplatform.nodeView.wallet._
 import org.ergoplatform.nodeView.wallet.requests._
@@ -155,12 +155,12 @@ case class WalletApiRoute(readersHolder: ActorRef,
   private def generateTransactionAndProcess(requests: Seq[TransactionGenerationRequest],
                                             inputsRaw: Seq[String],
                                             dataInputsRaw: Seq[String],
-                                            verifyFn: ErgoTransaction => Future[Try[ErgoTransaction]],
-                                            processFn: ErgoTransaction => Route): Route = {
-    withWalletOp(_.generateTransaction(requests, inputsRaw, dataInputsRaw).flatMap {
+                                            verifyFn: ErgoTransaction => Future[Try[UnconfirmedTransaction]],
+                                            processFn: UnconfirmedTransaction => Route): Route = {
+    withWalletOp(_.generateTransaction(requests, inputsRaw, dataInputsRaw).flatMap(txTry => txTry match {
       case Success(tx) => verifyFn(tx)
-      case f: Failure[ErgoTransaction] => Future(f)
-    }) {
+      case Failure(e) => Future(Failure[UnconfirmedTransaction](e))
+    })) {
       case Failure(e) => BadRequest(s"Bad request $requests. ${Option(e.getMessage).getOrElse(e.toString)}")
       case Success(tx) => processFn(tx)
     }
@@ -169,7 +169,13 @@ case class WalletApiRoute(readersHolder: ActorRef,
   private def generateTransaction(requests: Seq[TransactionGenerationRequest],
                                   inputsRaw: Seq[String],
                                   dataInputsRaw: Seq[String]): Route = {
-    generateTransactionAndProcess(requests, inputsRaw, dataInputsRaw, tx => Future(Success(tx)), tx => ApiResponse(tx))
+    generateTransactionAndProcess(
+      requests,
+      inputsRaw,
+      dataInputsRaw,
+      tx => Future(Success(UnconfirmedTransaction(tx))),
+      utx => ApiResponse(utx.transaction)
+    )
   }
 
   private def generateUnsignedTransaction(requests: Seq[TransactionGenerationRequest],