Merge pull request #1619 from ergoplatform/v4.0.24

Candidate for 4.0.24

.github/workflows/docker-publish.yml

@@ -30,4 +30,18 @@ jobs:
           context: .
           push: true
           tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Docker Metadata action for sigma-rust integration test node
+        uses: docker/[email protected]
+        id: meta
+        with:
+          images: ergoplatform/ergo-integration-test
+
+      - name: Build and push Docker images for integration-test node
+        uses: docker/[email protected]
+        with:
+          context: "{{defaultContext}}:sigma-rust-integration-test" 
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

README.md

@@ -82,7 +82,7 @@ To run specific Ergo version `<VERSION>` as a service with custom config `/path/
         -e MAX_HEAP=3G \
         ergoplatform/ergo:<VERSION> --<networkId> -c /etc/myergo.conf
 
-Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.23`.
+Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.24`.
 
 This will connect to the Ergo mainnet or testnet following your configuration passed in `myergo.conf` and network flag `--<networkId>`. Every default config value would be overwritten with corresponding value in `myergo.conf`. `MAX_HEAP` variable can be used to control how much memory can the node consume.
 

avldb/src/main/scala/scorex/db/KVStoreReader.scala

@@ -95,7 +95,7 @@ trait KVStoreReader extends AutoCloseable {
     * @param end - end of the range (inclusive)
     * @return
     */
-  def getRange(start: K, end: K): Seq[(K, V)] = {
+  def getRange(start: K, end: K, limit: Int = Int.MaxValue): Seq[(K, V)] = {
     val ro = new ReadOptions()
     ro.snapshot(db.getSnapshot)
     val iter = db.iterator(ro)
@@ -109,10 +109,12 @@ trait KVStoreReader extends AutoCloseable {
       }
       iter.seek(start)
       val bf = mutable.ArrayBuffer.empty[(K, V)]
-      while (iter.hasNext && check(iter.peekNext.getKey)) {
+      var elemCounter = 0
+      while (iter.hasNext && check(iter.peekNext.getKey) && elemCounter < limit) {
         val next = iter.next()
         val key = next.getKey
         val value = next.getValue
+        elemCounter += 1
         bf += (key -> value)
       }
       bf.toArray[(K,V)]

build.sbt

@@ -154,6 +154,7 @@ assemblyMergeStrategy in assembly := {
   case "logback.xml" => MergeStrategy.first
   case x if x.endsWith("module-info.class") => MergeStrategy.discard
   case "reference.conf" => CustomMergeStrategy.concatReversed
+  case PathList("org", "bouncycastle", xs @ _*) => MergeStrategy.first
   case PathList("org", "iq80", "leveldb", xs @ _*) => MergeStrategy.first
   case PathList("org", "bouncycastle", xs @ _*) => MergeStrategy.first
   case PathList("javax", "activation", xs @ _*) => MergeStrategy.last

ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/BoxSelector.scala

@@ -44,6 +44,12 @@ object BoxSelector {
   private val MinValuePerByteDefault = 30 * 12
   val MinBoxValue: Long = (MaxBoxSize.value / 2L) * MinValuePerByteDefault
 
+  /**
+    * Factor which is showing how many inputs selector is going through to optimize inputs.
+    * Bigger factor is slowing down inputs selection but minimizing chance of transaction failure.
+    */
+  val ScanDepthFactor = 300
+
   final case class BoxSelectionResult[T <: ErgoBoxAssets](boxes: Seq[T], changeBoxes: Seq[ErgoBoxAssets])
 
   trait BoxSelectionError {

ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala

@@ -25,6 +25,17 @@ object DefaultBoxSelector extends BoxSelector {
 
   final case class NotEnoughCoinsForChangeBoxesError(message: String) extends BoxSelectionError
 
+  // helper function which returns count of assets in `initialMap` not fully spent in `subtractor`
+  private def diffCount(initialMap: mutable.Map[ModifierId, Long], subtractor: TokensMap): Int = {
+    initialMap.foldLeft(0){case (cnt, (tokenId, tokenAmt)) =>
+      if (tokenAmt - subtractor.getOrElse(tokenId, 0L) > 0) {
+        cnt + 1
+      } else {
+        cnt
+      }
+    }
+  }
+
   override def select[T <: ErgoBoxAssets](inputBoxes: Iterator[T],
                                           externalFilter: T => Boolean,
                                           targetBalance: Long,
@@ -40,10 +51,30 @@ object DefaultBoxSelector extends BoxSelector {
       res += unspentBox
     }
 
-    def balanceMet = currentBalance >= targetBalance
+    /**
+      * Helper functions which checks whether enough ERGs collected
+      */
+    def balanceMet: Boolean = {
+      val diff = currentBalance - targetBalance
 
-    def assetsMet = targetAssets.forall {
-      case (id, targetAmt) => currentAssets.getOrElse(id, 0L) >= targetAmt
+      // We estimate how many ERG needed for assets in change boxes
+      val assetsDiff = diffCount(currentAssets, targetAssets)
+      val diffThreshold = if (assetsDiff <= 0) {
+        0
+      } else {
+        MinBoxValue * (assetsDiff / MaxAssetsPerBox + 1)
+      }
+
+      diff >= diffThreshold
+    }
+
+    /**
+      * Helper functions which checks whether enough assets collected
+      */
+    def assetsMet: Boolean = {
+      targetAssets.forall {
+        case (id, targetAmt) => currentAssets.getOrElse(id, 0L) >= targetAmt
+      }
     }
 
     @tailrec

ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ReplaceCompactCollectBoxSelector.scala

@@ -28,12 +28,6 @@ class ReplaceCompactCollectBoxSelector(maxInputs: Int, optimalInputs: Int) exten
 
   import ReplaceCompactCollectBoxSelector._
 
-  /**
-    * Factor which is showing how many input selector is going through to optimize inputs.
-    * Bigger factor is slowing down inputs selection but minimizing chance of transaction failure.
-    */
-  val ScanDepthFactor = 300
-
   /**
     * A method which is selecting boxes to spend in order to collect needed amounts of ergo tokens and assets.
     *
@@ -52,7 +46,7 @@ class ReplaceCompactCollectBoxSelector(maxInputs: Int, optimalInputs: Int) exten
                                           targetAssets: TokensMap): Either[BoxSelectionError, BoxSelectionResult[T]] = {
     // First picking up boxes in given order (1,2,3,4,...) by using DefaultBoxSelector
     DefaultBoxSelector.select(inputBoxes, filterFn, targetBalance, targetAssets).flatMapRight { initialSelection =>
-      val tail = inputBoxes.take(maxInputs * ScanDepthFactor).filter(filterFn).toSeq
+      val tail = inputBoxes.take(maxInputs * BoxSelector.ScanDepthFactor).filter(filterFn).toSeq
       // if number of inputs exceeds the limit, the selector is sorting remaining boxes(actually, only 10*maximum
       // boxes) by value in descending order and replaces small-value boxes in the inputs by big-value from the tail (1,2,3,4 => 10)
       (if (initialSelection.boxes.length > maxInputs) {

ergo-wallet/src/test/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelectorSpec.scala

@@ -8,11 +8,10 @@ import scorex.crypto.hash.{Blake2b256, Digest32}
 import sigmastate.Values
 import sigmastate.Values.SigmaPropValue
 import sigmastate.helpers.TestingHelpers._
-import scorex.util.{bytesToId, idToBytes}
+import scorex.util.{ModifierId, bytesToId, idToBytes}
 import org.scalatest.EitherValues
 import org.ergoplatform.wallet.boxes.DefaultBoxSelector.NotEnoughErgsError
 import org.ergoplatform.wallet.boxes.DefaultBoxSelector.NotEnoughTokensError
-import org.ergoplatform.wallet.boxes.DefaultBoxSelector.NotEnoughCoinsForChangeBoxesError
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.propspec.AnyPropSpec
 
@@ -24,10 +23,14 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues
 
   private val noFilter: TrackedBox => Boolean = _ => true
   private val onChainFilter = {box: TrackedBox => box.chainStatus.onChain}
-  val parentTx = ErgoLikeTransaction(IndexedSeq(), IndexedSeq())
+  private val parentTx = ErgoLikeTransaction(IndexedSeq(), IndexedSeq())
 
-  val TrueLeaf: SigmaPropValue = Values.TrueLeaf.toSigmaProp
-  val StartHeight: Int = 0
+  private val TrueLeaf: SigmaPropValue = Values.TrueLeaf.toSigmaProp
+  private val StartHeight: Int = 0
+
+  private def genTokens(count: Int) = {
+    (0 until count).map { i => Digest32 @@ idToBytes(bytesToId(Blake2b256(i.toString))) -> i.toLong }
+  }
 
   property("returns error when it is impossible to select coins") {
     val box = testBox(1, TrueLeaf, creationHeight = StartHeight)
@@ -163,19 +166,17 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues
     val s1 = select(uBoxes.toIterator, noFilter, 1 * MinBoxValue, Map(assetId3 -> 11))
     s1 shouldBe 'right
 
-    s1.right.get.boxes.size shouldBe 2
-    s1.right.get.boxes should contain theSameElementsAs(Seq(uBox1, uBox3))
+    s1.right.get.boxes.size shouldBe 3
+    s1.right.get.boxes should contain theSameElementsAs(Seq(uBox1, uBox2, uBox3))
 
     s1.right.get.changeBoxes.size shouldBe 1
-    s1.right.get.changeBoxes(0).value shouldBe 100 * MinBoxValue
+    s1.right.get.changeBoxes(0).value shouldBe 110 * MinBoxValue
     s1.right.get.changeBoxes(0).tokens(assetId1) shouldBe 1
     s1.right.get.changeBoxes(0).tokens(assetId2) shouldBe 1
     s1.right.get.changeBoxes(0).tokens(assetId3) shouldBe 90
     s1.right.get.changeBoxes(0).tokens(assetId4) shouldBe 101
-    s1.right.get.changeBoxes(0).tokens(assetId5) shouldBe 100
-    s1.right.get.changeBoxes(0).tokens(assetId6) shouldBe 100
-
-    s1.right.get.boxes shouldBe Seq(uBox1, uBox3)
+    s1.right.get.changeBoxes(0).tokens(assetId5) shouldBe 110
+    s1.right.get.changeBoxes(0).tokens(assetId6) shouldBe 110
 
     val s2 = select(uBoxes.toIterator, noFilter, 10 * MinBoxValue,
       Map(assetId1 -> 1, assetId2 -> 1, assetId3 -> 1, assetId4 -> 1))
@@ -186,8 +187,7 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues
     s2.right.get.changeBoxes(0).tokens(assetId7) shouldBe 10
     s2.right.get.changeBoxes(0).tokens(assetId8) shouldBe 10
 
-    //todo: should selector fail in this case (if there's no monetary value to create a new box w. assets) ?
-    select(uBoxes.toIterator, noFilter, 1 * MinBoxValue, Map(assetId1 -> 1)).left.value shouldBe a [NotEnoughCoinsForChangeBoxesError]
+    select(uBoxes.toIterator, noFilter, 1 * MinBoxValue, Map(assetId1 -> 1)).isRight shouldBe true
   }
 
   property("Size of a box with MaxAssetsPerBox tokens should not cross MaxBoxSize") {
@@ -199,11 +199,8 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues
   }
 
   property("Select boxes such that change boxes are grouped by MaxAssetsPerBox") {
-    def genTokens(count: Int) =
-      (0 until count).map { i => Digest32 @@ idToBytes(bytesToId(Blake2b256(i.toString))) -> i.toLong }
-
     // make selection such that '2 * MaxAssetsPerBox + 1' tokens generates exactly 2 change boxes with MaxAssetsPerBox tokens
-    val box1 = testBox(3 * MinBoxValue, TrueLeaf, StartHeight, genTokens(2 * MaxAssetsPerBox + 1))
+    val box1 = testBox(4 * MinBoxValue, TrueLeaf, StartHeight, genTokens(2 * MaxAssetsPerBox + 1))
     val uBox1 = TrackedBox(parentTx, 0, Some(100), box1, Set(PaymentsScanId))
     val s1 = select(Iterator(uBox1), noFilter, 1 * MinBoxValue, Map(bytesToId(Blake2b256("1")) -> 1))
     s1 shouldBe 'right
@@ -218,4 +215,37 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues
     s2.right.get.changeBoxes.size shouldBe 3
     s2.right.get.changeBoxes.exists(_.tokens.size == 1) shouldBe true
   }
+
+  // test which shows that https://github.com/ergoplatform/ergo/issues/1644 fixed
+  property("i1644: should collect needed inputs when needed for change in presence of assets") {
+    val tokenData = genTokens(3).last
+    tokenData._2 shouldBe 2
+
+    val tokenId = ModifierId @@ bytesToId(tokenData._1)
+
+    val ergValue = 10 * MinBoxValue
+
+    val box1 = testBox(ergValue, TrueLeaf, StartHeight, Seq(tokenData))
+    val uBox1 = TrackedBox(parentTx, 0, Some(100), box1, Set(PaymentsScanId))
+    val box2 = testBox(ergValue, TrueLeaf, StartHeight)
+    val uBox2 = TrackedBox(parentTx, 0, Some(100), box2, Set(PaymentsScanId))
+
+    val s1 = select(Iterator(uBox1, uBox2), noFilter, ergValue, Map.empty)
+    s1 shouldBe 'right
+    s1.right.get.changeBoxes.size shouldBe 1
+    s1.right.get.changeBoxes.head.tokens(tokenId) shouldBe 2
+
+    val box3 = testBox(ergValue, TrueLeaf, StartHeight)
+    val uBox3 = TrackedBox(parentTx, 0, Some(100), box3, Set(PaymentsScanId))
+
+    val s2 = select(Iterator(uBox2, uBox3), noFilter, ergValue, Map.empty)
+    s2 shouldBe 'right
+    s2.right.get.changeBoxes.size shouldBe 0
+
+    val s3 = select(Iterator(uBox1, uBox2), noFilter, ergValue, Map(tokenId -> 1))
+    s3 shouldBe 'right
+    s3.right.get.changeBoxes.size shouldBe 1
+
+  }
+
 }

sigma-rust-integration-test/Dockerfile

@@ -0,0 +1,3 @@
+FROM ergoplatform/ergo:latest
+COPY ergo.conf /etc/myergo.conf
+ENTRYPOINT ["java", "-jar", "/home/ergo/ergo.jar", "--devnet", "-c", "/etc/myergo.conf"]

sigma-rust-integration-test/ergo.conf

@@ -0,0 +1,104 @@
+ergo {
+  # Settings for node view holder regime. See papers.yellow.ModifiersProcessing.md
+  node {
+    # State type.  Possible options are:
+    # "utxo" - keep full utxo set, that allows to validate arbitrary block and generate ADProofs
+    # "digest" - keep state root hash only and validate transactions via ADProofs
+    stateType = "utxo"
+
+    # Download block transactions and verify them (requires BlocksToKeep == 0 if disabled)
+    verifyTransactions = true
+
+    # Number of last blocks to keep with transactions and ADproofs, for all other blocks only header will be stored.
+    # Keep all blocks from genesis if negative
+    blocksToKeep = -1
+
+    # Download PoPoW proof on node bootstrap
+    PoPoWBootstrap = false
+
+    # Minimal suffix size for PoPoW proof (may be pre-defined constant or settings parameter)
+    minimalSuffix = 10
+
+    # Is the node is doing mining
+    mining = true
+
+    # Use external miner, native miner is used if set to `false`
+    useExternalMiner = false
+
+    # If true, a node generate blocks being offline. The only really useful case for it probably is to start a new
+    # blockchain
+    offlineGeneration = true
+
+    # internal miner's interval of polling for a candidate
+    internalMinerPollingInterval = 5s
+
+    mempoolCapacity = 10000
+  }
+
+  chain {
+    # Difficulty network start with
+    initialDifficultyHex = "01"
+
+    powScheme {
+      powType = "autolykos"
+      k = 32
+      n = 26
+    }
+
+  }
+
+  wallet {
+
+    secretStorage {
+
+      secretDir = ${ergo.directory}"/wallet/keystore"
+
+      encryption {
+
+        # Pseudo-random function with output of length `dkLen` (PBKDF2 param)
+        prf = "HmacSHA256"
+
+        # Number of PBKDF2 iterations (PBKDF2 param)
+        c = 128000
+
+        # Desired bit-length of the derived key (PBKDF2 param)
+        dkLen = 256
+      }
+
+    }
+
+    # Generating seed length in bits
+    # Options: 128, 160, 192, 224, 256
+    seedStrengthBits = 160
+
+    # Language to be used in mnemonic seed
+    # Options: "chinese_simplified", "chinese_traditional", "english", "french", "italian", "japanese", "korean", "spanish"
+    mnemonicPhraseLanguage = "english"
+
+    defaultTransactionFee = 10000
+
+    # Save used boxes (may consume additinal disk space) or delete them immediately
+    keepSpentBoxes = false
+
+    # Mnemonic seed used in wallet for tests
+    testMnemonic = "ozone drill grab fiber curtain grace pudding thank cruise elder eight picnic"
+
+    # Number of keys to be generated for tests
+    testKeysQty = 5
+  }
+}
+
+scorex {
+  network {
+    maxPacketSize = 2048576
+    maxInvObjects = 400
+    bindAddress = "0.0.0.0:9001"
+    knownPeers = []
+    agentName = "ergo-integration-test"
+  }
+  restApi {
+    bindAddress = "0.0.0.0:9053"
+    # Hash of "hello", taken from mainnet config
+    apiKeyHash = "324dcf027dd4a30a932c441f365a25e86b173defa4b8e58948253471b81b72cf"
+  }
+}