Merge pull request #2022 from ergoplatform/i2014

Flag that a peer was bootstrapped via UTXO set snapshot

src/main/resources/application.conf

@@ -42,9 +42,6 @@ ergo {
         # how many different proofs we are downloading from other peers
         # and comparing with each other, before choosing best one
         p2pNipopows = 1
-
-        # Minimal suffix size for NiPoPoW proof (may be pre-defined constant or settings parameter)
-        nipopowSuffix = 10
     }
 
     # Is the node is doing mining

src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala

@@ -35,7 +35,6 @@ import scorex.util.{ModifierId, ScorexLogging}
 import scorex.core.network.DeliveryTracker
 import scorex.core.network.peer.PenaltyType
 import scorex.core.transaction.state.TransactionValidation.TooHighCostError
-import scorex.core.app.Version
 import scorex.crypto.hash.Digest32
 import org.ergoplatform.nodeView.state.UtxoState.{ManifestId, SubtreeId}
 import org.ergoplatform.ErgoLikeContext.Height
@@ -75,8 +74,6 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
       Restart
   }
 
-  private val blockSectionsDownloadFilter = BlockSectionsDownloadFilter(settings.nodeSettings.stateType)
-
   private var syncInfoV1CacheByHeadersHeight: Option[(Int, ErgoSyncInfoV1)] = Option.empty
 
   private var syncInfoV2CacheByHeadersHeight: Option[(Int, ErgoSyncInfoV2)] = Option.empty
@@ -530,7 +527,8 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
     * @return available peers to download headers from together with the state/origin of the peer
     */
   private def getPeersForDownloadingHeaders(callingPeer: ConnectedPeer): Iterable[ConnectedPeer] = {
-    syncTracker.peersByStatus.getOrElse(Older, Array(callingPeer))
+    val nonFiltered: Iterable[ConnectedPeer] = syncTracker.peersByStatus.getOrElse(Older, Array(callingPeer))
+    HeadersDownloadFilter.filter(nonFiltered)
   }
 
   /**
@@ -545,7 +543,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
       .orElse {
         Option(peersByStatus.getOrElse(Unknown, mutable.WrappedArray.empty) ++ peersByStatus.getOrElse(Fork, mutable.WrappedArray.empty))
           .filter(_.nonEmpty)
-      }.map(blockSectionsDownloadFilter.filter)
+      }.map(BlockSectionsDownloadFilter.filter)
   }
 
   /**
@@ -1129,14 +1127,8 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
           Seq.empty
         }
       case _ =>
-        if (peer.peerInfo.map(_.peerSpec.protocolVersion).getOrElse(Version.initial) == Version.v4043 &&
-          modifierTypeId == Header.modifierTypeId) {
-          log.debug("Header ids from 4.0.43")
-          Seq.empty
-        } else {
-          log.info(s"Processing ${invData.ids.length} non-tx invs (of type $modifierTypeId) from $peer")
-          invData.ids.filter(mid => deliveryTracker.status(mid, modifierTypeId, Seq(hr)) == ModifiersStatus.Unknown)
-        }
+        log.info(s"Processing ${invData.ids.length} non-tx invs (of type $modifierTypeId) from $peer")
+        invData.ids.filter(mid => deliveryTracker.status(mid, modifierTypeId, Seq(hr)) == ModifiersStatus.Unknown)
     }
 
     if (newModifierIds.nonEmpty) {

src/main/scala/org/ergoplatform/network/ModePeerFeature.scala

@@ -15,36 +15,73 @@ import scorex.util.serialization.{Reader, Writer}
   *
   * @param stateType - information on whether UTXO set is store (so state type is UTXO/Digest)
   * @param verifyingTransactions - whether the peer is verifying transactions
-  * @param nipopowSuffix - whether the peer has has bootstrapped via Nipopows, and length of proof suffix
+  * @param nipopowBootstrapped - whether the peer has bootstrapped via Nipopows,
+  *                              and if so, supported bootstrapping options (only one currently)
   * @param blocksToKeep - how many last full blocks the peer is storing
   */
 case class ModePeerFeature(stateType: StateType,
                            verifyingTransactions: Boolean,
-                           nipopowSuffix: Option[Int],
+                           nipopowBootstrapped: Option[Int],
                            blocksToKeep: Int) extends PeerFeature {
   override type M = ModePeerFeature
 
   override val featureId: Id = PeerFeatureDescriptors.ModeFeatureId
 
   override def serializer: ErgoSerializer[ModePeerFeature] = ModeFeatureSerializer
+
+  /**
+    * @return whether the peer has all the full blocks
+    */
+  def allBlocksAvailable: Boolean = blocksToKeep == ModePeerFeature.AllBlocksKept
+
+
+  /**
+    * @return whether the peer has all the headers
+    */
+  def allHeadersAvailable: Boolean = nipopowBootstrapped.isEmpty
 }
 
 object ModePeerFeature {
 
   import io.circe.syntax._
 
+  /**
+    * Flag which is indicating NiPoPoW bootstrap mode. Currently there is only one option (suffix proof done
+    * according to KMZ17 paper), which does not exclude possibility for more options in future
+    */
+  val NiPoPoWDefaultFlag = 1
+
+  /**
+    * Flag value which is when used as length of blockchain suffix kept locally means that all the full blocks are
+    * stored
+    */
+  val AllBlocksKept = -1
+
+  /**
+    * Flag value which is when used as length of blockchain suffix kept locally means that a node was bootstrapped
+    * via UTXO set snapshot, so not all the full blocks stored, but at the same time there is no fixed-length suffix
+    * as after bootstrapping there is no pruning
+    */
+  val UTXOSetBootstrapped = -2
+
   def apply(nodeSettings: NodeConfigurationSettings): ModePeerFeature = {
-    val popowSuffix = if (nodeSettings.nipopowSettings.nipopowBootstrap) {
-      Some(nodeSettings.nipopowSettings.nipopowSuffix)
+    val popowBootstrapped = if (nodeSettings.nipopowSettings.nipopowBootstrap) {
+      Some(NiPoPoWDefaultFlag)
     } else {
       None
     }
 
+    val blocksKept = if (nodeSettings.utxoSettings.utxoBootstrap) {
+      UTXOSetBootstrapped
+    } else {
+      nodeSettings.blocksToKeep
+    }
+
     new ModePeerFeature(
       nodeSettings.stateType,
       nodeSettings.verifyTransactions,
-      popowSuffix,
-      nodeSettings.blocksToKeep
+      popowBootstrapped,
+      blocksKept
     )
   }
 
@@ -76,22 +113,22 @@ object ModeFeatureSerializer extends ErgoSerializer[ModePeerFeature] {
   override def serialize(mf: ModePeerFeature, w: Writer): Unit = {
     w.put(mf.stateType.stateTypeCode)
     w.put(booleanToByte(mf.verifyingTransactions))
-    w.putOption(mf.nipopowSuffix)(_.putInt(_))
-    w.putInt(mf.blocksToKeep) // todo: put -2 if bootstrapped via utxo set snapshot? https://github.com/ergoplatform/ergo/issues/2014
+    w.putOption(mf.nipopowBootstrapped)(_.putInt(_))
+    w.putInt(mf.blocksToKeep)
   }
 
   override def parse(r: Reader): ModePeerFeature = {
     require(r.remaining < MaxSize)
 
     val stateType = StateType.fromCode(r.getByte())
     val verifyingTransactions = byteToBoolean(r.getByte())
-    val popowSuffix = r.getOption(r.getInt())
+    val popowBootstrapped = r.getOption(r.getInt())
     val blocksToKeep = r.getInt()
 
     new ModePeerFeature(
       stateType,
       verifyingTransactions,
-      popowSuffix,
+      popowBootstrapped,
       blocksToKeep
     )
   }

src/main/scala/org/ergoplatform/network/VersionBasedPeerFilteringRule.scala

@@ -1,6 +1,5 @@
 package org.ergoplatform.network
 
-import org.ergoplatform.nodeView.state.StateType
 import scorex.core.app.Version
 import scorex.core.network.ConnectedPeer
 
@@ -48,44 +47,6 @@ trait VersionBasedPeerFilteringRule extends PeerFilteringRule {
 
 }
 
-
-/**
-  * 4.0.22+ allow for downloading ADProofs that are too big in block at 667614
-  * for prior versions, a peer will not deliver block # 667614 and some other blocks
-  */
-object DigestModeFilter extends VersionBasedPeerFilteringRule {
-
-  override def condition(version: Version): Boolean = {
-      version.compare(Version.v4022) >= 0
-  }
-
-}
-
-/**
-  * Filter out peers of 4.0.17 or 4.0.18 version as they are delivering broken block sections
-  */
-object BrokenModifiersFilter extends VersionBasedPeerFilteringRule {
-
-  override def condition(version: Version): Boolean = {
-    version != Version.v4017 && version != Version.v4018
-  }
-
-}
-
-/**
-  * Filter to download block sections, combining `DigestModeFilter` and `BrokenModifiersFilter`
-  * @param stateType - own (node's) state type
-  */
-final case class BlockSectionsDownloadFilter(stateType: StateType) extends VersionBasedPeerFilteringRule {
-  override def condition(version: Version): Boolean = {
-    if (stateType == StateType.Digest) {
-      DigestModeFilter.condition(version)
-    } else {
-      BrokenModifiersFilter.condition(version)
-    }
-  }
-}
-
 /**
   * If peer's version is >= 4.0.16, the peer is supporting sync V2
   */
@@ -125,8 +86,29 @@ object NipopowSupportFilter extends PeerFilteringRule {
   override def condition(peer: ConnectedPeer): Boolean = {
     val version = peer.peerInfo.map(_.peerSpec.protocolVersion).getOrElse(Version.initial)
 
-    peer.mode.flatMap(_.nipopowSuffix).isEmpty &&
+    peer.mode.flatMap(_.nipopowBootstrapped).isEmpty &&
       version.compare(Version.NipopowActivationVersion) >= 0
   }
 
 }
+
+/**
+  * Filter to download block sections (except of headers).
+  * Currently, it is accepting peers which do have all the full blocks (so not bootstrapped via UTXO set snapshot
+  * or stateless client with full blocks suffix)
+  */
+object BlockSectionsDownloadFilter extends PeerFilteringRule {
+  override def condition(peer: ConnectedPeer): Boolean = {
+    peer.mode.exists(_.allBlocksAvailable)
+  }
+}
+
+/**
+  * Filter to download headers.
+  * Currently, it is accepting peers which do have all the headers (so not bootstrapped via NiPoPoWs.
+  */
+object HeadersDownloadFilter extends PeerFilteringRule {
+  override def condition(peer: ConnectedPeer): Boolean = {
+    peer.mode.exists(_.allHeadersAvailable)
+  }
+}

src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala

@@ -42,7 +42,7 @@ trait UtxoSettingsReader {
 /**
   * Settings related to headers-chain bootstrapping with NiPoPoWs. See ergo.node.nipopow section for settings description.
   */
-case class NipopowSettings(nipopowBootstrap: Boolean, p2pNipopows: Int, nipopowSuffix: Int)
+case class NipopowSettings(nipopowBootstrap: Boolean, p2pNipopows: Int)
 
 /**
   * Custom settings reader for `NipopowSettings`
@@ -51,8 +51,7 @@ trait NipopowSettingsReader {
   implicit val nipopowSettingsReader: ValueReader[NipopowSettings] = { (cfg, path) =>
     NipopowSettings(
       cfg.as[Boolean](s"$path.nipopowBootstrap"),
-      cfg.as[Int](s"$path.p2pNipopows"),
-      cfg.as[Int](s"$path.nipopowSuffix")
+      cfg.as[Int](s"$path.p2pNipopows")
     )
   }
 }

src/main/scala/scorex/core/app/Version.scala

@@ -33,14 +33,6 @@ object Version {
 
   val initial: Version = Version(0, 0, 1)
 
-  val v4017: Version = Version(4, 0, 17)
-
-  val v4018: Version = Version(4, 0, 18)
-
-  val v4022: Version = Version(4, 0, 22)
-
-  val v4043: Version = Version(4, 0, 43)
-
   val Eip37ForkVersion: Version = Version(4, 0, 100)
   val JitSoftForkVersion: Version = Version(5, 0, 0)
 

src/test/scala/org/ergoplatform/network/PeerFilteringRuleSpecification.scala

@@ -25,25 +25,6 @@ class PeerFilteringRuleSpecification extends ErgoPropertyTest {
     SyncV2Filter.filter(Seq(v1Peer, v2Peer0, v2Peer1, v2Peer2)) shouldBe Seq(v2Peer0, v2Peer1, v2Peer2)
   }
 
-  property("digest mode filter") {
-    val beforePeer = peerWithVersion(Version(4, 0, 21))
-    val afterPeer0 = peerWithVersion(Version(4, 0, 22))
-    val afterPeer1 = peerWithVersion(Version(5, 0, 0))
-
-    DigestModeFilter.filter(Seq(beforePeer, afterPeer0, afterPeer1)) shouldBe Seq(afterPeer0, afterPeer1)
-  }
-
-  property("broken modifiers filter") {
-    val inPeer0 = peerWithVersion(Version(4, 0, 17))
-    val inPeer1 = peerWithVersion(Version(4, 0, 18))
-    val outPeer0 = peerWithVersion(Version(4, 0, 16))
-    val outPeer1 = peerWithVersion(Version(4, 0, 19))
-    val outPeer2 = peerWithVersion(Version(5, 0, 0))
-
-    BrokenModifiersFilter.filter(Seq(inPeer0, inPeer1, outPeer0, outPeer1, outPeer2)) shouldBe
-      Seq(outPeer0, outPeer1, outPeer2)
-  }
-
   property("utxo set snapshot filter") {
     val peer0 = peerWithVersion(Version(4, 0, 17))
     val peer1 = peerWithVersion(Version(4, 0, 18))

src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala

@@ -38,7 +38,7 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w
   override protected implicit val addressEncoder: ErgoAddressEncoder = initSettings.chainSettings.addressEncoder
 
   val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(StateType.Utxo, verifyTransactions = true,
-    -1, UtxoSettings(false, 0, 2), NipopowSettings(false, 1, ChainGenerator.minimalSuffix), mining = false, ChainGenerator.txCostLimit, ChainGenerator.txSizeLimit, useExternalMiner = false,
+    -1, UtxoSettings(false, 0, 2), NipopowSettings(false, 1), mining = false, ChainGenerator.txCostLimit, ChainGenerator.txSizeLimit, useExternalMiner = false,
     internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false,
     200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 20,
     1000000, 100, adProofsSuffixLength = 112 * 1024, extraIndex = false)

src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala

@@ -25,7 +25,7 @@ class ErgoSettingsSpecification extends ErgoPropertyTest {
       verifyTransactions = true,
       1000,
       utxoSettings = UtxoSettings(false, 0, 2),
-      nipopowSettings = NipopowSettings(false, 1, 10),
+      nipopowSettings = NipopowSettings(false, 1),
       mining = true,
       txCostLimit,
       txSizeLimit,
@@ -74,7 +74,7 @@ class ErgoSettingsSpecification extends ErgoPropertyTest {
       verifyTransactions = true,
       12,
       utxoSettings = UtxoSettings(false, 0, 2),
-      nipopowSettings = NipopowSettings(false, 1, 10),
+      nipopowSettings = NipopowSettings(false, 1),
       mining = true,
       txCostLimit,
       txSizeLimit,
@@ -116,7 +116,7 @@ class ErgoSettingsSpecification extends ErgoPropertyTest {
       verifyTransactions = true,
       13,
       utxoSettings = UtxoSettings(false, 0, 2),
-      nipopowSettings = NipopowSettings(false, 1, 10),
+      nipopowSettings = NipopowSettings(false, 1),
       mining = true,
       txCostLimit,
       txSizeLimit,

src/test/scala/org/ergoplatform/tools/ChainGenerator.scala

@@ -59,7 +59,7 @@ object ChainGenerator extends App with ErgoTestHelpers {
   val txCostLimit     = initSettings.nodeSettings.maxTransactionCost
   val txSizeLimit     = initSettings.nodeSettings.maxTransactionSize
   val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(StateType.Utxo, verifyTransactions = true,
-    -1, UtxoSettings(false, 0, 2), NipopowSettings(false, 1, minimalSuffix), mining = false, txCostLimit, txSizeLimit, useExternalMiner = false,
+    -1, UtxoSettings(false, 0, 2), NipopowSettings(false, 1), mining = false, txCostLimit, txSizeLimit, useExternalMiner = false,
     internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false,
     200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 20,
     1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false)

src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala

@@ -44,11 +44,10 @@ trait HistoryTestHelpers extends ErgoPropertyTest {
                       initialDiffOpt: Option[BigInt] = None,
                       genesisIdOpt: Option[ModifierId] = None): ErgoHistory = {
 
-    val minimalSuffix = 2
     val txCostLimit     = initSettings.nodeSettings.maxTransactionCost
     val txSizeLimit     = initSettings.nodeSettings.maxTransactionSize
     val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(stateType, verifyTransactions, blocksToKeep,
-      UtxoSettings(false, 0, 2), NipopowSettings(false, 1, minimalSuffix), mining = false, txCostLimit, txSizeLimit, useExternalMiner = false,
+      UtxoSettings(false, 0, 2), NipopowSettings(false, 1), mining = false, txCostLimit, txSizeLimit, useExternalMiner = false,
       internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None,
       offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte,
       rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false

src/test/scala/org/ergoplatform/utils/NodeViewTestConfig.scala

@@ -18,7 +18,7 @@ case class NodeViewTestConfig(stateType: StateType,
       nodeSettings = defaultSettings.nodeSettings.copy(
         stateType = stateType,
         verifyTransactions = verifyTransactions,
-        nipopowSettings = NipopowSettings(popowBootstrap, 1, 10)
+        nipopowSettings = NipopowSettings(popowBootstrap, 1)
       )
     )
   }

src/test/scala/org/ergoplatform/utils/Stubs.scala

@@ -366,11 +366,10 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with
                       epochLength: Int = 100000000,
                       useLastEpochs: Int = 10): ErgoHistory = {
 
-    val minimalSuffix = 2
     val txCostLimit     = initSettings.nodeSettings.maxTransactionCost
     val txSizeLimit      = initSettings.nodeSettings.maxTransactionSize
     val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(stateType, verifyTransactions, blocksToKeep,
-      UtxoSettings(false, 0, 2), NipopowSettings(poPoWBootstrap, 1, minimalSuffix), mining = false, txCostLimit, txSizeLimit, useExternalMiner = false,
+      UtxoSettings(false, 0, 2), NipopowSettings(poPoWBootstrap, 1), mining = false, txCostLimit, txSizeLimit, useExternalMiner = false,
       internalMinersCount = 1, internalMinerPollingInterval = 1.second,miningPubKeyHex = None,
       offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte,
       rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false