Merge pull request #1927 from ergoplatform/utxo-set-bootstrapping-pre…

…paration

Preliminary functions for bootstrapping with UTXO set snapshot #1

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/NodeParameters.scala

@@ -1,4 +1,9 @@
 package scorex.crypto.authds.avltree.batch
 
-//todo: support arbitrary-size values
+/**
+  * Parameters of AVL+ tree nodes (internal and leaves)
+  * @param keySize - size of a key (fixed)
+  * @param valueSize - size of a value in a leaf (fixed, if defined, arbitrary, if None)
+  * @param labelSize - size of a label (node hash), fixed
+  */
 case class NodeParameters(keySize: Int, valueSize: Option[Int], labelSize: Int)

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProxyInternalProverNode.scala

@@ -1,33 +1,41 @@
 package scorex.crypto.authds.avltree.batch
 
-
 import scorex.crypto.authds.{ADKey, Balance}
 import scorex.crypto.hash.{CryptographicHash, Digest}
 import scorex.db.LDBVersionedStore
 import InternalNode.InternalNodePrefix
 
+/**
+  * Internal node where children are not provided during node construction, only pointers to them,
+  * and then children nodes are read from database and constructed only when requested (and children internal nodes are
+  * of the same type). It allows for lazy loading of a tree.
+  *
+  */
 class ProxyInternalProverNode[D <: Digest](protected var pk: ADKey,
-                                           val lkey: ADKey,
-                                           val rkey: ADKey,
+                                           val leftLabel: ADKey,
+                                           val rightLabel: ADKey,
                                            protected var pb: Balance = Balance @@ 0.toByte)
                                           (implicit val phf: CryptographicHash[D],
                                            store: LDBVersionedStore,
                                            nodeParameters: NodeParameters)
   extends InternalProverNode(k = pk, l = null, r = null, b = pb)(phf) {
 
   override protected def computeLabel: D = {
-    hf.hash(Array(InternalNodePrefix, b), lkey, rkey)
+    hf.hash(Array(InternalNodePrefix, b), leftLabel, rightLabel)
   }
 
   override def left: ProverNodes[D] = {
-    if (l == null) l = VersionedLDBAVLStorage.fetch[D](lkey)
+    if (l == null) {
+      l = VersionedLDBAVLStorage.fetch[D](leftLabel)
+    }
     l
   }
 
   override def right: ProverNodes[D] = {
-    if (r == null) r = VersionedLDBAVLStorage.fetch[D](rkey)
+    if (r == null) {
+      r = VersionedLDBAVLStorage.fetch[D](rightLabel)
+    }
     r
   }
-}
-
 
+}

avldb/src/main/scala/scorex/db/LDBVersionedStore.scala

@@ -1,16 +1,13 @@
 package scorex.db
 
 import java.io.File
-
 import scorex.db.LDBFactory.factory
 import org.iq80.leveldb._
-import java.nio.ByteBuffer
 
+import java.nio.ByteBuffer
 import scala.collection.mutable.ArrayBuffer
 import java.util.concurrent.locks.ReentrantReadWriteLock
-
 import scorex.crypto.hash.Blake2b256
-
 import scala.util.Try
 
 

src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala

@@ -73,12 +73,20 @@ case class Header(override val version: Header.Version,
   override def minerPk: EcPointType = powSolution.pk
 
   /**
-    * Expected identifiers of the block sections
+    * Expected identifiers of the block sections corresponding to this header
     */
-  lazy val sectionIds: Seq[(ModifierTypeId, ModifierId)] = Seq(
-    (ADProofs.modifierTypeId, ADProofsId),
-    (BlockTransactions.modifierTypeId, transactionsId),
-    (Extension.modifierTypeId, extensionId))
+  lazy val sectionIds: Seq[(ModifierTypeId, ModifierId)] =
+    Array(
+      (ADProofs.modifierTypeId, ADProofsId),
+      (BlockTransactions.modifierTypeId, transactionsId),
+      (Extension.modifierTypeId, extensionId)
+    )
+
+  /**
+    * Expected identifiers of the block sections corresponding to this header,
+    * except of state transformations proof section id
+    */
+  lazy val sectionIdsWithNoProof: Seq[(ModifierTypeId, ModifierId)] = sectionIds.tail
 
   override lazy val toString: String = s"Header(${this.asJson.noSpaces})"
 

src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala

@@ -6,11 +6,9 @@ import org.ergoplatform.ErgoLikeContext
 import org.ergoplatform.mining.AutolykosPowScheme
 import org.ergoplatform.modifiers.history._
 import org.ergoplatform.modifiers.history.header.{Header, PreGenesisHeader}
-import org.ergoplatform.modifiers.state.UTXOSnapshotChunk
 import org.ergoplatform.modifiers.{NonHeaderBlockSection, ErgoFullBlock, BlockSection}
 import org.ergoplatform.nodeView.history.storage.HistoryStorage
 import org.ergoplatform.nodeView.history.storage.modifierprocessors._
-import org.ergoplatform.nodeView.history.storage.modifierprocessors.popow.{EmptyPoPoWProofsProcessor, FullPoPoWProofsProcessor}
 import org.ergoplatform.settings._
 import org.ergoplatform.utils.LoggingUtil
 import scorex.core.consensus.ProgressInfo
@@ -79,10 +77,6 @@ trait ErgoHistory
           process(header)
         case section: NonHeaderBlockSection =>
           process(section)
-        case poPoWProof: NipopowProofModifier =>
-          process(poPoWProof)
-        case chunk: UTXOSnapshotChunk =>
-          process(chunk)
       }
     }.map(this -> _).recoverWith { case e =>
       if (!e.isInstanceOf[RecoverableModifierError]) {
@@ -281,35 +275,31 @@ object ErgoHistory extends ScorexLogging {
 
     val history: ErgoHistory = (nodeSettings.verifyTransactions, nodeSettings.poPoWBootstrap) match {
       case (true, true) =>
-        new ErgoHistory with FullBlockSectionProcessor
-          with FullPoPoWProofsProcessor {
+        new ErgoHistory with FullBlockSectionProcessor {
           override protected val settings: ErgoSettings = ergoSettings
           override protected[history] val historyStorage: HistoryStorage = db
           override val powScheme: AutolykosPowScheme = chainSettings.powScheme
           override protected val timeProvider: NetworkTimeProvider = ntp
         }
 
       case (false, true) =>
-        new ErgoHistory with EmptyBlockSectionProcessor
-          with FullPoPoWProofsProcessor {
+        new ErgoHistory with EmptyBlockSectionProcessor {
           override protected val settings: ErgoSettings = ergoSettings
           override protected[history] val historyStorage: HistoryStorage = db
           override val powScheme: AutolykosPowScheme = chainSettings.powScheme
           override protected val timeProvider: NetworkTimeProvider = ntp
         }
 
       case (true, false) =>
-        new ErgoHistory with FullBlockSectionProcessor
-          with EmptyPoPoWProofsProcessor {
+        new ErgoHistory with FullBlockSectionProcessor {
           override protected val settings: ErgoSettings = ergoSettings
           override protected[history] val historyStorage: HistoryStorage = db
           override val powScheme: AutolykosPowScheme = chainSettings.powScheme
           override protected val timeProvider: NetworkTimeProvider = ntp
         }
 
       case (false, false) =>
-        new ErgoHistory with EmptyBlockSectionProcessor
-          with EmptyPoPoWProofsProcessor {
+        new ErgoHistory with EmptyBlockSectionProcessor {
           override protected val settings: ErgoSettings = ergoSettings
           override protected[history] val historyStorage: HistoryStorage = db
           override val powScheme: AutolykosPowScheme = chainSettings.powScheme

src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala

@@ -4,12 +4,10 @@ import org.ergoplatform.modifiers.history._
 import org.ergoplatform.modifiers.history.extension.Extension
 import org.ergoplatform.modifiers.history.header.{Header, PreGenesisHeader}
 import org.ergoplatform.modifiers.history.popow.{NipopowAlgos, NipopowProof, PoPowHeader, PoPowParams}
-import org.ergoplatform.modifiers.state.UTXOSnapshotChunk
 import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NonHeaderBlockSection}
 import org.ergoplatform.nodeView.history.ErgoHistory.Height
 import org.ergoplatform.nodeView.history.storage._
 import org.ergoplatform.nodeView.history.storage.modifierprocessors._
-import org.ergoplatform.nodeView.history.storage.modifierprocessors.popow.PoPoWProofsProcessor
 import org.ergoplatform.settings.ErgoSettings
 import scorex.core.{ModifierTypeId, NodeViewComponent}
 import scorex.core.consensus.{ContainsModifiers, Equal, Fork, ModifierSemanticValidity, Older, PeerChainStatus, Unknown, Younger}
@@ -28,8 +26,6 @@ trait ErgoHistoryReader
   extends NodeViewComponent
     with ContainsModifiers[BlockSection]
     with HeadersProcessor
-    with PoPoWProofsProcessor
-    with UTXOSnapshotChunkProcessor
     with BlockSectionProcessor
     with ScorexLogging
     with ScorexEncoding {
@@ -425,10 +421,6 @@ trait ErgoHistoryReader
         validate(header)
       case m: NonHeaderBlockSection =>
         validate(m)
-      case m: NipopowProofModifier =>
-        validate(m)
-      case chunk: UTXOSnapshotChunk =>
-        validate(chunk)
       case m: Any =>
         Failure(new MalformedModifierError(s"Modifier $m has incorrect type", modifier.id, modifier.modifierTypeId))
     }

src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala

@@ -123,7 +123,7 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging {
     } else if (nodeSettings.stateType.requireProofs) {
       h.sectionIds
     } else {
-      h.sectionIds.tail // do not download UTXO set transformation proofs if UTXO set is stored
+      h.sectionIdsWithNoProof // do not download UTXO set transformation proofs if UTXO set is stored
     }
   }