diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a835a2110a..17d49677ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - scala: [2.12.10, 2.11.12] + scala: [2.13.8, 2.12.10, 2.11.12] java: [adopt@1.8] runs-on: ${{ matrix.os }} steps: diff --git a/.gitignore b/.gitignore index 1deb9ac892..626004c1c7 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ local.conf .metals/ .vscode/ project/metals.sbt +null/ # scala worksheets *.worksheet.sc \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 1c5f95e3c4..95b719cb24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM sbtscala/scala-sbt:eclipse-temurin-11.0.15_1.6.2_2.12.16 as builder +FROM sbtscala/scala-sbt:eclipse-temurin-11.0.15_1.7.1_2.13.8 as builder WORKDIR /mnt COPY build.sbt findbugs-exclude.xml ./ COPY project/ project/ diff --git a/README.md b/README.md index 0f602c16ee..d145829dc5 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,17 @@ heavy validation attacks A [White Paper](https://ergoplatform.org/docs/whitepaper.pdf) with a brief description is available. A Yellow Paper with detailed specification is underway and will be available shortly. At the moment, there are [drafts of the Yellow Paper](https://github.com/ergoplatform/ergo/tree/master/papers/yellow) available, and currently the reference implementation code should be considered as the specification. +## Security assumptions + +This client relies on some assumptions in regards with its environment: + +* execution environment is trusted. While seed is stored in encrypted files, and the client's + wallet tries to remove secret key from memory as soon as possible when it is not needed, the + client has no protection from side-channel attacks, memory scans etc. +* clocks should be more or less synchronized. If timestamp of a block is more than 20 minutes + in future, the block will be temporarily rejected. The client does not use NTP or other time + syncing protocols. + ## Building and Running Node and UI See [documentation](https://docs.ergoplatform.com/node/install/) diff --git a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala index a3e1ead7af..2648c36245 100644 --- a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala +++ b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala @@ -12,7 +12,7 @@ object Helper { type HF = Blake2b256.type type Prover = PersistentBatchAVLProver[Digest32, HF] - implicit val hf = Blake2b256 + implicit val hf: HF = Blake2b256 val kl = 32 val vl = 8 diff --git a/avldb/build.sbt b/avldb/build.sbt index 7d3e95dba6..3378dd64ee 100644 --- a/avldb/build.sbt +++ b/avldb/build.sbt @@ -16,7 +16,8 @@ libraryDependencies ++= Seq( ) libraryDependencies ++= Seq( - "org.ethereum" % "leveldbjni-all" % "1.18.3" + "org.ethereum" % "leveldbjni-all" % "1.18.3", + "org.typelevel" %% "spire" % "0.14.1" ) testOptions in Test := Seq(Tests.Filter(t => !t.matches(".*Benchmark$"))) diff --git a/avldb/src/main/scala/scorex/db/LDBKVStore.scala b/avldb/src/main/scala/scorex/db/LDBKVStore.scala index bdf3ade613..ffd53609db 100644 --- a/avldb/src/main/scala/scorex/db/LDBKVStore.scala +++ b/avldb/src/main/scala/scorex/db/LDBKVStore.scala @@ -4,6 +4,7 @@ import org.iq80.leveldb.DB import scorex.util.ScorexLogging import scala.util.{Failure, Success, Try} +import spire.syntax.all.cfor /** @@ -13,11 +14,13 @@ import scala.util.{Failure, Success, Try} */ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging { - def update(toInsert: Seq[(K, V)], toRemove: Seq[K]): Try[Unit] = { + def update(toInsert: Array[(K, V)], toRemove: Array[K]): Try[Unit] = { val batch = db.createWriteBatch() + val insertLen = toInsert.length + val removeLen = toRemove.length try { - toInsert.foreach { case (k, v) => batch.put(k, v) } - toRemove.foreach(batch.delete) + cfor(0)(_ < insertLen, _ + 1) { i => batch.put(toInsert(i)._1, toInsert(i)._2)} + cfor(0)(_ < removeLen, _ + 1) { i => batch.delete(toRemove(i))} db.write(batch) Success(()) } catch { @@ -42,9 +45,9 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging } } - def insert(values: Seq[(K, V)]): Try[Unit] = update(values, Seq.empty) + def insert(values: Array[(K, V)]): Try[Unit] = update(values, Array.empty) - def remove(keys: Seq[K]): Try[Unit] = update(Seq.empty, keys) + def remove(keys: Array[K]): Try[Unit] = update(Array.empty, keys) /** * Get last key within some range (inclusive) by used comparator. diff --git a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala b/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala index 63dba526e2..2791f8f03c 100644 --- a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala +++ b/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala @@ -228,9 +228,12 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e Undo(versionID, key, value) } + /** + * Write versioned batch update to the database, removing keys from the database and adding new key -> value pairs + */ def update(versionID: VersionID, - toRemove: Iterable[Array[Byte]], - toUpdate: Iterable[(Array[Byte], Array[Byte])]): Try[Unit] = Try { + toRemove: TraversableOnce[Array[Byte]], + toUpdate: TraversableOnce[(Array[Byte], Array[Byte])]): Try[Unit] = Try { lock.writeLock().lock() val lastLsn = lsn // remember current LSN value val batch = db.createWriteBatch() diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala index ecadcd1ad5..1067650c2f 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala @@ -15,7 +15,7 @@ object BatchingBenchmark extends App with FileHelper { val NumMods = 200000 - implicit val hf = Blake2b256 + implicit val hf: HF = Blake2b256 type HF = Blake2b256.type val store = new LDBVersionedStore(getRandomTempDir, initialKeepVersions = 10) diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/BenchRunner.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/BenchRunner.scala index c8fbcbc157..2dfdfbfd0e 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/bench/BenchRunner.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/bench/BenchRunner.scala @@ -10,12 +10,8 @@ import org.ergoplatform.nodeView.{ErgoNodeViewRef, NVBenchmark} import org.ergoplatform.settings.{Args, ErgoSettings} import org.ergoplatform.nodeView.ErgoNodeViewHolder.CurrentView import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{GetDataFromCurrentView, LocallyGeneratedModifier} -import scorex.core.utils.{NetworkTimeProvider, NetworkTimeProviderSettings} import scorex.util.ScorexLogging - import scala.concurrent.ExecutionContextExecutor -import scala.concurrent.duration._ -import scala.language.postfixOps object BenchRunner extends ScorexLogging with NVBenchmark { @@ -41,10 +37,7 @@ object BenchRunner extends ScorexLogging with NVBenchmark { log.info(s"Setting that being used:") log.info(s"$ergoSettings") - val ntpSettings = NetworkTimeProviderSettings("pool.ntp.org", 30 minutes, 30 seconds) - val timeProvider = new NetworkTimeProvider(ntpSettings) - - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) /** * It's a hack to set minimalFullBlockHeightVar to 0 and to avoid "Header Is Not Synced" error, cause diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala index 027bafa170..04e0540995 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala @@ -5,11 +5,8 @@ import java.io.FileOutputStream import org.ergoplatform.bench.misc.ModifierWriter import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.settings.{Args, ErgoSettings} -import scorex.core.settings.ScorexSettings -import scorex.core.utils.NetworkTimeProvider import scorex.util.ScorexLogging -import scala.concurrent.ExecutionContext.Implicits.global object HistoryExtractor extends ScorexLogging { @@ -18,11 +15,9 @@ object HistoryExtractor extends ScorexLogging { lazy val cfgPath: Option[String] = args.headOption lazy val outputFile: String = args.lift(1).getOrElse("blocks.dat") lazy val ergoSettings: ErgoSettings = ErgoSettings.read(Args(cfgPath, None)) - lazy val settings: ScorexSettings = ergoSettings.scorexSettings - val timeProvider = new NetworkTimeProvider(settings.ntp) val os = new FileOutputStream(outputFile) - val h = ErgoHistory.readOrGenerate(ergoSettings, timeProvider) + val h = ErgoHistory.readOrGenerate(ergoSettings)(null) val wholeChain = h.chainToHeader(None, h.bestHeaderOpt.get) var counter = 0 diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/misc/ModifierWriter.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/misc/ModifierWriter.scala index 3f448d9125..1372cab088 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/bench/misc/ModifierWriter.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/bench/misc/ModifierWriter.scala @@ -1,18 +1,17 @@ package org.ergoplatform.bench.misc import java.io.{InputStream, OutputStream} - import com.google.common.primitives.Ints import org.ergoplatform.Utils._ -import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.modifiers.{BlockSection, NetworkObjectTypeId} import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} import scorex.core.serialization.ScorexSerializer -import scorex.core.{ModifierTypeId, NodeViewModifier} +import scorex.core.NodeViewModifier object ModifierWriter { - val modifierSerializers: Map[ModifierTypeId, ScorexSerializer[_ <: BlockSection]] = + val modifierSerializers: Map[NetworkObjectTypeId.Value, ScorexSerializer[_ <: BlockSection]] = Map(Header.modifierTypeId -> HeaderSerializer, BlockTransactions.modifierTypeId -> BlockTransactionsSerializer, ADProofs.modifierTypeId -> ADProofsSerializer) @@ -34,9 +33,9 @@ object ModifierWriter { mod <- modifierSerializers(typeId).parseBytesTry(bytes).toOption } yield mod - private def readModId(implicit fis: InputStream): Option[ModifierTypeId] = { + private def readModId(implicit fis: InputStream): Option[NetworkObjectTypeId.Value] = { val int = fis.read() - if (int == -1) { None } else { Some(ModifierTypeId @@ int.toByte) } + if (int == -1) { None } else { Some(NetworkObjectTypeId.fromByte(int.toByte)) } } } diff --git a/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala b/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala index 2467856d46..652bb40982 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala @@ -37,7 +37,7 @@ object LDBStoreBench } val txsWithDbGen: Gen[(Seq[BlockTransactions], LDBKVStore)] = txsGen.map { bts => - val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes) + val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes).toArray val db = storeLDB() toInsert.grouped(5).foreach(db.insert(_).get) bts -> storeLDB @@ -53,7 +53,7 @@ object LDBStoreBench private def randomVersion: Digest32 = Algos.hash(Longs.toByteArray(Random.nextLong())) private def benchWriteLDB(bts: Seq[BlockTransactions]): Unit = { - val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes) + val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes).toArray val db = storeLDB() toInsert.grouped(5).foreach(db.insert(_).get) } diff --git a/build.sbt b/build.sbt index 41806f983f..18b62c9859 100644 --- a/build.sbt +++ b/build.sbt @@ -6,6 +6,7 @@ logLevel := Level.Debug // this values should be in sync with ergo-wallet/build.sbt val scala211 = "2.11.12" val scala212 = "2.12.10" +val scala213 = "2.13.8" lazy val commonSettings = Seq( organization := "org.ergoplatform", @@ -36,7 +37,7 @@ val circeVersion = "0.13.0" val akkaVersion = "2.6.10" val akkaHttpVersion = "10.2.4" -val sigmaStateVersion = "5.0.3" +val sigmaStateVersion = "5.0.5" // for testing current sigmastate build (see sigmastate-ergo-it jenkins job) val effectiveSigmaStateVersion = Option(System.getenv().get("SIGMASTATE_VERSION")).getOrElse(sigmaStateVersion) @@ -61,7 +62,6 @@ libraryDependencies ++= Seq( "com.typesafe.akka" %% "akka-parsing" % akkaHttpVersion, "com.typesafe.akka" %% "akka-stream" % akkaVersion, "org.bitlet" % "weupnp" % "0.1.4", - "commons-net" % "commons-net" % "3.6", // api dependencies "io.circe" %% "circe-core" % circeVersion, @@ -265,7 +265,7 @@ lazy val avldb_benchmarks = (project in file("avldb/benchmarks")) lazy val ergoWallet = (project in file("ergo-wallet")) .disablePlugins(ScapegoatSbtPlugin) // not compatible with crossScalaVersions .settings( - crossScalaVersions := Seq(scalaVersion.value, scala211), + crossScalaVersions := Seq(scala213, scalaVersion.value, scala211), commonSettings, name := "ergo-wallet", libraryDependencies ++= Seq( diff --git a/ergo-wallet/build.sbt b/ergo-wallet/build.sbt index dcb251032e..e9dcf17c62 100644 --- a/ergo-wallet/build.sbt +++ b/ergo-wallet/build.sbt @@ -1,12 +1,13 @@ // this values should be in sync with root (i.e. ../build.sbt) val scala211 = "2.11.12" val scala212 = "2.12.10" +val scala213 = "2.13.8" val circeVersion = "0.13.0" val circeVersion211 = "0.10.0" libraryDependencies ++= Seq( - "org.scodec" %% "scodec-bits" % "1.1.6", + "org.scodec" %% "scodec-bits" % "1.1.34", "io.circe" %% "circe-core" % (if (scalaVersion.value == scala211) circeVersion211 else circeVersion), "io.circe" %% "circe-generic" % (if (scalaVersion.value == scala211) circeVersion211 else circeVersion), diff --git a/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java b/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java index 6f1f7dda5d..4493e490e5 100644 --- a/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java +++ b/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java @@ -2,7 +2,7 @@ import org.ergoplatform.ErgoLikeTransaction; import org.ergoplatform.UnsignedErgoLikeTransaction; -import scala.collection.JavaConversions; +import scala.collection.JavaConverters; import sigmastate.basics.DLogProtocol; import java.util.Map; @@ -26,8 +26,10 @@ public ErgoLikeTransaction prove(UnsignedErgoLikeTransaction unsignedTx, DLogPro * @return signed transaction */ public ErgoLikeTransaction prove(UnsignedErgoLikeTransaction unsignedTx, Map sks) { - // JavaConversions used to support Scala 2.11 - return org.ergoplatform.wallet.interpreter.ErgoUnsafeProver.prove(unsignedTx, JavaConversions.mapAsScalaMap(sks)); + // This method of JavaConverters is supported across Scala 2.11-2.13 + return org.ergoplatform.wallet.interpreter.ErgoUnsafeProver.prove( + unsignedTx, + JavaConverters.mapAsScalaMapConverter(sks).asScala()); } } diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala index cdc5f07c2e..534f55dc49 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala @@ -104,7 +104,7 @@ class DefaultBoxSelector(override val reemissionDataOpt: Option[ReemissionData]) assetsMet )) { formChangeBoxes(currentBalance, targetBalance, currentAssets, targetAssets).mapRight { changeBoxes => - selectionResultWithEip27Output(res, changeBoxes) + selectionResultWithEip27Output(res.toSeq, changeBoxes) } } else { Left(NotEnoughTokensError( diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala index 9d4a1f1330..e7e55d5288 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala @@ -3,6 +3,8 @@ package org.ergoplatform.wallet.boxes import org.ergoplatform.ErgoBoxCandidate import sigmastate.eval.Extensions._ import java7.compat.Math + +import scala.collection.compat.immutable.ArraySeq import scala.collection.mutable import scala.util.Try @@ -29,7 +31,7 @@ object ErgoBoxAssetExtractor { ) box.additionalTokens.foreach { case (assetId, amount) => - val aiWrapped = mutable.WrappedArray.make(assetId) + val aiWrapped = ArraySeq.unsafeWrapArray(assetId) val total = map.getOrElse(aiWrapped, 0L) map.put(aiWrapped, Math.addExact(total, amount)) } diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala index 700e5f48af..88db0570c4 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala @@ -66,8 +66,8 @@ object Mnemonic { * Converts mnemonic phrase to seed it was derived from. */ def toSeed(mnemonic: SecretString, passOpt: Option[SecretString] = None): Array[Byte] = { - val normalizedMnemonic = normalize(mnemonic.getData(), NFKD).toCharArray - val normalizedPass = normalize(("mnemonic".toCharArray ++ passOpt.fold("".toCharArray())(_.getData())), NFKD) + val normalizedMnemonic = normalize(ArrayCharSequence(mnemonic.getData()), NFKD).toCharArray + val normalizedPass = normalize(ArrayCharSequence("mnemonic".toCharArray ++ passOpt.fold("".toCharArray())(_.getData())), NFKD) passOpt.fold(())(_.erase()) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala index 0fbde5068d..e8dedb0c9e 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala @@ -234,7 +234,7 @@ object TransactionBuilder { // add burnTokens to target assets so that they are excluded from the change outputs // thus total outputs assets will be reduced which is interpreted as _token burning_ - val tokensOutWithBurned = AssetUtils.mergeAssets(tokensOutNoMinted, burnTokens) + val tokensOutWithBurned = AssetUtils.mergeAssets(tokensOutNoMinted.toMap, burnTokens) val selection = boxSelector.select(inputs.toIterator, outputTotal, tokensOutWithBurned) match { case Left(err) => throw new IllegalArgumentException( diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/mnemonic/MnemonicSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/mnemonic/MnemonicSpec.scala index 19a4a339d0..60db9060c8 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/mnemonic/MnemonicSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/mnemonic/MnemonicSpec.scala @@ -404,7 +404,7 @@ class MnemonicSpec val strength = Mnemonic.AllowedStrengths.zip(Mnemonic.MnemonicSentenceSizes).find(_._2 == sentenceSize).map(_._1).get val mnemonic = new Mnemonic(langId, strength) Base16.encode(Mnemonic.toSeed(SecretString.create(sentence), Some(SecretString.create(pass)))) shouldEqual seed - normalize(mnemonic.toMnemonic(Base16.decode(entropy).get).get.getData(), NFKD) shouldEqual normalize(sentence, NFKD) + normalize(ArrayCharSequence(mnemonic.toMnemonic(Base16.decode(entropy).get).get.getData()), NFKD) shouldEqual normalize(sentence, NFKD) } } diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala index 9fc73b1398..ec4f7ff88f 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala @@ -15,7 +15,6 @@ import sigmastate.basics.DLogProtocol.ProveDlog import sigmastate.{SByte, SType} import org.ergoplatform.wallet.Constants.{ScanId, PaymentsScanId} import scorex.util._ -import scala.collection.IndexedSeq import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBoxCandidate import org.ergoplatform.ErgoScriptPredef diff --git a/src/it/scala/org/ergoplatform/it/WalletSpec.scala b/src/it/scala/org/ergoplatform/it/WalletSpec.scala index 9c511d014c..a3c6b4012a 100644 --- a/src/it/scala/org/ergoplatform/it/WalletSpec.scala +++ b/src/it/scala/org/ergoplatform/it/WalletSpec.scala @@ -79,8 +79,8 @@ class WalletSpec extends AsyncWordSpec with IntegrationSuite with WalletTestOps val encodedBox = Base16.encode(ErgoBoxSerializer.toBytes(input)) - val paymentRequest = PaymentRequest(P2PKAddress(pk), 50000000, Seq.empty, Map.empty) - val requestsHolder = RequestsHolder(Seq(paymentRequest), feeOpt = Some(100000L), Seq(encodedBox), dataInputsRaw = Seq.empty, minerRewardDelay = 720) + val paymentRequest = PaymentRequest(P2PKAddress(pk)(addressEncoder), 50000000, Seq.empty, Map.empty) + val requestsHolder = RequestsHolder(Seq(paymentRequest), feeOpt = Some(100000L), Seq(encodedBox), dataInputsRaw = Seq.empty, minerRewardDelay = 720)(addressEncoder) node.waitForStartup.flatMap { node: Node => for { diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index f193638740..94981d26ed 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "5.0.4" + version: "5.0.7" title: Ergo Node API description: API docs for Ergo Node. Models are shared between all Ergo products contact: @@ -200,6 +200,129 @@ components: example: 1 default: 1 + BalanceInfo: + type: object + description: Balance information + required: + - nanoErgs + - tokens + properties: + nanoErgs: + type: integer + format: int64 + description: Balance of nanoERGs + tokens: + type: array + description: Balance of tokens + items: + type: object + properties: + tokenId: + $ref: '#/components/schemas/ModifierId' + description: Identifier of the token + amount: + type: integer + format: int64 + description: Amount of the token + decimals: + type: integer + description: Number of decimals of the token + name: + type: string + description: Name of the token, if any + + IndexedErgoBox: + type: object + description: Box indexed with extra information + required: + - box + - confirmationsNum + - address + - creationTransaction + - spendingTransaction + - spendingHeight + - inclusionHeight + - spent + - globalIndex + properties: + box: + $ref: '#/components/schemas/ErgoTransactionOutput' + confirmationsNum: + description: Number of confirmations, if the box is included into the blockchain + type: integer + format: int32 + minimum: 0 + example: 147 + nullable: true + address: + $ref: '#/components/schemas/ErgoAddress' + creationTransaction: + description: Transaction which created the box + $ref: '#/components/schemas/ModifierId' + spendingTransaction: + description: Transaction which created the box + nullable: true + $ref: '#/components/schemas/ModifierId' + spendingHeight: + description: The height the box was spent at + type: integer + format: int32 + minimum: 0 + example: 147 + nullable: true + inclusionHeight: + description: The height the transaction containing the box was included in a block at + type: integer + format: int32 + minimum: 0 + example: 147 + spent: + description: A flag signalling whether the box was spent + type: boolean + example: false + globalIndex: + description: Global index of the output in the blockchain + type: integer + format: int64 + minimum: 0 + example: 83927 + + IndexedToken: + type: object + description: Token indexed with extra information + required: + -id + -boxId + -emissionAmount + -name + -description + -decimals + properties: + id: + description: Id of the token + $ref: '#/components/schemas/ModifierId' + boxId: + description: Id of the box that created the token + $ref: '#/components/schemas/ModifierId' + emissionAmount: + description: The total supply of the token + type: integer + format: int64 + minimum: 1 + example: 3500000 + name: + description: The name of the token + type: string + description: + description: The description of the token + type: string + decimals: + description: The number of decimals the token supports + type: integer + format: int32 + minimum: 0 + example: 8 + UnsignedErgoTransaction: type: object description: Unsigned Ergo transaction @@ -306,6 +429,70 @@ components: type: integer format: int32 + IndexedErgoTransaction: + type: object + description: Transaction indexed with extra information + required: + - id + - inputs + - dataInputs + - outputs + - inclusionHeight + - numConfirmations + - blockId + - timestamp + - index + - globalIndex + - size + properties: + id: + $ref: '#/components/schemas/TransactionId' + inputs: + description: Transaction inputs + type: array + items: + $ref: '#/components/schemas/ErgoTransactionInput' + dataInputs: + description: Transaction data inputs + type: array + items: + $ref: '#/components/schemas/ErgoTransactionDataInput' + outputs: + description: Transaction outputs + type: array + items: + $ref: '#/components/schemas/ErgoTransactionOutput' + inclusionHeight: + description: Height of a block the transaction was included in + type: integer + format: int32 + example: 20998 + numConfirmations: + description: Number of transaction confirmations + type: integer + format: int32 + example: 20998 + blockId: + description: Id of the block the transaction was included in + allOf: + - $ref: '#/components/schemas/ModifierId' + timestamp: + $ref: '#/components/schemas/Timestamp' + index: + description: index of the transaction in the block it was included in + type: integer + format: int32 + example: 3 + globalIndex: + description: Global index of the transaction in the blockchain + type: integer + format: int64 + example: 3565445 + size: + description: Size in bytes + type: integer + format: int32 + ErgoAddress: description: Encoded Ergo Address type: string @@ -5454,6 +5641,589 @@ paths: application/json: schema: $ref: '#/components/schemas/EmissionScripts' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/indexedHeight: + get: + summary: Get current block height the indexer is at + operationId: getIndexedHeight + tags: + - blockchain + responses: + '200': + description: height of the indexer and full height + content: + application/json: + schema: + properties: + indexedHeight: + type: integer + default: 0 + description: number of blocks indexed + fullHeight: + type: integer + description: number of all known blocks + + /blockchain/transaction/byId/{txId}: + get: + summary: Retrieve a transaction by its id + operationId: getTxById + tags: + - blockchain + parameters: + - in: path + name: txId + required: true + description: id of the wanted transaction + schema: + type: string + responses: + '200': + description: transaction with wanted id + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedErgoTransaction' + '404': + description: Transaction with this id doesn't exist + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/transaction/byIndex/{txIndex}: + get: + summary: Retrieve a transaction by global index number + operationId: getTxByIndex + tags: + - blockchain + parameters: + - in: path + name: txIndex + required: true + description: index of the wanted transaction + schema: + type: number + responses: + '200': + description: transaction with wanted index + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedErgoTransaction' + '404': + description: Transaction with this index doesn't exist + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/transaction/byAddress: + post: + summary: Retrieve transactions by their associated address + operationId: getTxsByAddress + tags: + - blockchain + requestBody: + required: true + content: + application/json: + description: adderess associated with transactions + schema: + $ref: '#/components/schemas/ErgoAddress' + parameters: + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int32 + default: 0 + - in: query + name: limit + required: false + description: amount of elements to retrieve + schema: + type: integer + format: int32 + default: 5 + responses: + '200': + description: transactions associated with wanted address + content: + application/json: + schema: + type: object + properties: + items: + type: array + description: Array of transactions + items: + $ref: '#/components/schemas/IndexedErgoTransaction' + total: + type: integer + description: Total count of retreived transactions + '404': + description: No transactions found for wanted address + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/transaction/range: + get: + summary: Get a range of transaction ids + operationId: getTxRange + tags: + - blockchain + parameters: + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int32 + default: 0 + - in: query + name: limit + required: false + description: amount of elements to retrieve + schema: + type: integer + format: int32 + default: 5 + responses: + '200': + description: transactions ids in wanted range + content: + application/json: + schema: + type: array + description: Array of transaction ids + items: + $ref: '#/components/schemas/ModifierId' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/box/byId/{boxId}: + get: + summary: Retrieve a box by its id + operationId: getBoxById + tags: + - blockchain + parameters: + - in: path + name: boxId + required: true + description: id of the wanted box + schema: + type: string + responses: + '200': + description: box with wanted id + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedErgoBox' + '404': + description: No box found with wanted id + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/box/byIndex/{boxIndex}: + get: + summary: Retrieve a box by global index number + operationId: getBoxByIndex + tags: + - blockchain + parameters: + - in: path + name: boxIndex + required: true + description: index of the wanted box + schema: + type: number + responses: + '200': + description: box with wanted index + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedErgoBox' + '404': + description: Box with this index doesn't exist + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/box/byAddress: + post: + summary: Retrieve boxes by their associated address + operationId: getBoxesByAddress + tags: + - blockchain + requestBody: + required: true + content: + application/json: + description: adderess associated with boxes + schema: + $ref: '#/components/schemas/ErgoAddress' + parameters: + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int32 + default: 0 + - in: query + name: limit + required: false + description: amount of elements to retrieve + schema: + type: integer + format: int32 + default: 5 + responses: + '200': + description: boxes associated with wanted address + content: + application/json: + schema: + type: object + properties: + items: + type: array + description: Array of boxes + items: + $ref: '#/components/schemas/IndexedErgoTransaction' + total: + type: integer + description: Total number of retreived boxes + '404': + description: No boxes found for wanted address + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/box/unspent/byAddress: + post: + summary: Retrieve unspent boxes by their associated address + operationId: getBoxesByAddressUnspent + tags: + - blockchain + requestBody: + required: true + content: + application/json: + description: adderess associated with unspent boxes + schema: + $ref: '#/components/schemas/ErgoAddress' + parameters: + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int32 + default: 0 + - in: query + name: limit + required: false + description: amount of elements to retrieve + schema: + type: integer + format: int32 + default: 5 + responses: + '200': + description: unspent boxes associated with wanted address + content: + application/json: + schema: + type: array + description: Array of boxes + items: + $ref: '#/components/schemas/IndexedErgoTransaction' + '404': + description: No unspent boxes found for wanted address + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/box/range: + get: + summary: Get a range of box ids + operationId: getBoxRange + tags: + - blockchain + parameters: + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int32 + default: 0 + - in: query + name: limit + required: false + description: amount of elements to retrieve + schema: + type: integer + format: int32 + default: 5 + responses: + '200': + description: box ids in wanted range + content: + application/json: + schema: + type: array + description: Array of box ids + items: + $ref: '#/components/schemas/ModifierId' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/box/byErgoTree: + post: + summary: Retrieve boxes by their associated ergotree + operationId: getBoxesByErgoTree + tags: + - blockchain + requestBody: + required: true + content: + application/json: + description: hex encoded ergotree + schema: + type: string + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' + parameters: + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int32 + default: 0 + - in: query + name: limit + required: false + description: amount of elements to retrieve + schema: + type: integer + format: int32 + default: 5 + responses: + '200': + description: boxes with wanted ergotree + content: + application/json: + schema: + type: object + properties: + items: + type: array + description: Array of boxes + items: + $ref: '#/components/schemas/IndexedErgoBox' + total: + type: integer + description: Total number of retreived boxes + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/box/unspent/byErgoTree: + post: + summary: Retrieve unspent boxes by their associated ergotree + operationId: getBoxesByErgoTreeUnspent + tags: + - blockchain + requestBody: + required: true + content: + application/json: + description: hex encoded ergotree + schema: + type: string + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' + parameters: + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int32 + default: 0 + - in: query + name: limit + required: false + description: amount of elements to retrieve + schema: + type: integer + format: int32 + default: 5 + responses: + '200': + description: unspent boxes with wanted ergotree + content: + application/json: + schema: + type: object + properties: + items: + type: array + description: Array of boxes + items: + $ref: '#/components/schemas/IndexedErgoBox' + total: + type: integer + description: Total number of retreived boxes + '404': + description: No unspent box found with wanted ergotree + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/token/byId/{tokenId}: + get: + summary: Retrieve minting information about a token + operationId: getTokenById + tags: + - blockchain + parameters: + - in: path + name: tokenId + required: true + description: id of the wanted token + schema: + type: string + responses: + '200': + description: token with wanted id + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedToken' + '404': + description: No token found with wanted id + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/balance: + post: + summary: Retrieve confirmed and unconfirmed balance of an address + operationId: getAddressBalanceTotal + tags: + - blockchain + requestBody: + required: true + content: + application/json: + description: adderess with balance + schema: + $ref: '#/components/schemas/ErgoAddress' + responses: + '200': + description: balance information + content: + application/json: + schema: + type: object + properties: + confirmed: + $ref: '#/components/schemas/BalanceInfo' + unconfirmed: + $ref: '#/components/schemas/BalanceInfo' default: description: Error content: diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 319ad12a33..b10464f45f 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -74,6 +74,9 @@ ergo { # Minimal fee amount of transactions mempool accepts minimalFeeAmount = 1000000 + # If true, the node will store all transactions, boxes and addresses in an index. + extraIndex = false + # List with hex-encoded identifiers of transactions banned from getting into memory pool blacklistedTransactions = [] @@ -111,6 +114,9 @@ ergo { # Number of recently used block sections that will be kept in memory blockSectionsCacheSize = 12 + # Number of recently used extra indexes that will be kept in memory (has no effect if extraIndex is disabled) + extraCacheSize = 500 + # Number of recently used headers that will be kept in memory headersCacheSize = 1000 @@ -396,7 +402,7 @@ scorex { nodeName = "ergo-node" # Network protocol version to be sent in handshakes - appVersion = 5.0.6 + appVersion = 5.0.7 # Network agent name. May contain information about client code # stack, starting from core code-base up to the end graphical interface. @@ -539,16 +545,6 @@ scorex { peerDiscovery = true } - ntp { - # NTP server address - server = "pool.ntp.org" - - # update time rate - updateEvery = 30m - - # server answer timeout - timeout = 30s - } } diff --git a/src/main/resources/mainnet.conf b/src/main/resources/mainnet.conf index 4c03289d9b..e84cb490f2 100644 --- a/src/main/resources/mainnet.conf +++ b/src/main/resources/mainnet.conf @@ -80,7 +80,7 @@ scorex { network { magicBytes = [1, 0, 2, 4] bindAddress = "0.0.0.0:9030" - nodeName = "ergo-mainnet-5.0.6" + nodeName = "ergo-mainnet-5.0.7" nodeName = ${?NODENAME} knownPeers = [ "213.239.193.208:9030", diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index 54d26d41ed..ca0cbf6545 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -11,6 +11,7 @@ import org.ergoplatform.mining.ErgoMiner import org.ergoplatform.mining.ErgoMiner.StartMining import org.ergoplatform.network.{ErgoNodeViewSynchronizer, ErgoSyncTracker} import org.ergoplatform.nodeView.history.ErgoSyncInfoMessageSpec +import org.ergoplatform.nodeView.history.extra.ExtraIndexer import org.ergoplatform.nodeView.{ErgoNodeViewRef, ErgoReadersHolderRef} import org.ergoplatform.settings.{Args, ErgoSettings, NetworkType} import scorex.core.api.http._ @@ -21,13 +22,16 @@ import scorex.core.network.message.Message.MessageCode import scorex.core.network.message._ import scorex.core.network.peer.PeerManagerRef import scorex.core.settings.ScorexSettings -import scorex.core.utils.NetworkTimeProvider import scorex.util.ScorexLogging import java.net.InetSocketAddress import scala.concurrent.{ExecutionContext, Future} -import scala.io.Source +import scala.io.{Codec, Source} +/** + * Ergo reference protocol client application runnable from command line + * @param args parsed command line arguments + */ class ErgoApp(args: Args) extends ScorexLogging { log.info(s"Running with args: $args") @@ -50,8 +54,6 @@ class ErgoApp(args: Args) extends ScorexLogging { implicit private val executionContext: ExecutionContext = actorSystem.dispatcher - private val timeProvider = new NetworkTimeProvider(scorexSettings.ntp) - private val upnpGateway: Option[UPnPGateway] = if (scorexSettings.network.upnpEnabled) UPnP.getValidGateway(scorexSettings.network) else None @@ -80,25 +82,32 @@ class ErgoApp(args: Args) extends ScorexLogging { private val scorexContext = ScorexContext( messageSpecs = basicSpecs ++ additionalMessageSpecs, upnpGateway = upnpGateway, - timeProvider = timeProvider, externalNodeAddress = externalSocketAddress ) private val peerManagerRef = PeerManagerRef(ergoSettings, scorexContext) - private val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + private val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) private val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) // Create an instance of ErgoMiner actor if "mining = true" in config private val minerRefOpt: Option[ActorRef] = if (ergoSettings.nodeSettings.mining) { - Some(ErgoMiner(ergoSettings, nodeViewHolderRef, readersHolderRef, timeProvider)) + Some(ErgoMiner(ergoSettings, nodeViewHolderRef, readersHolderRef)) } else { None } - private val syncTracker = ErgoSyncTracker(scorexSettings.network, timeProvider) + if(ergoSettings.nodeSettings.extraIndex) + require( + ergoSettings.nodeSettings.stateType.holdsUtxoSet && !ergoSettings.nodeSettings.isFullBlocksPruned, + "Node must store full UTXO set and all blocks to run extra indexer." + ) + // Create an instance of ExtraIndexer actor (will start if "extraIndex = true" in config) + ExtraIndexer(ergoSettings.chainSettings, ergoSettings.cacheSettings) + + private val syncTracker = ErgoSyncTracker(scorexSettings.network) private val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(ergoSettings) @@ -107,7 +116,6 @@ class ErgoApp(args: Args) extends ScorexLogging { nodeViewHolderRef, ErgoSyncInfoMessageSpec, ergoSettings, - timeProvider, syncTracker, deliveryTracker ) @@ -152,30 +160,30 @@ class ErgoApp(args: Args) extends ScorexLogging { readersHolderRef, networkControllerRef, syncTracker, - ergoSettings, - timeProvider + ergoSettings ) private val apiRoutes: Seq[ApiRoute] = Seq( - EmissionApiRoute(ergoSettings), - ErgoUtilsApiRoute(ergoSettings), - ErgoPeersApiRoute( - peerManagerRef, - networkControllerRef, - syncTracker, - deliveryTracker, - scorexSettings.restApi - ), - InfoApiRoute(statsCollectorRef, scorexSettings.restApi, timeProvider), - BlocksApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings), - NipopowApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings), - TransactionsApiRoute(readersHolderRef, nodeViewHolderRef, ergoSettings), - WalletApiRoute(readersHolderRef, nodeViewHolderRef, ergoSettings), - UtxoApiRoute(readersHolderRef, scorexSettings.restApi), - ScriptApiRoute(readersHolderRef, ergoSettings), - ScanApiRoute(readersHolderRef, ergoSettings), - NodeApiRoute(ergoSettings) - ) ++ minerRefOpt.map(minerRef => MiningApiRoute(minerRef, ergoSettings)).toSeq + EmissionApiRoute(ergoSettings), + ErgoUtilsApiRoute(ergoSettings), + BlockchainApiRoute(readersHolderRef, ergoSettings), + ErgoPeersApiRoute( + peerManagerRef, + networkControllerRef, + syncTracker, + deliveryTracker, + scorexSettings.restApi + ), + InfoApiRoute(statsCollectorRef, scorexSettings.restApi), + BlocksApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings), + NipopowApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings), + TransactionsApiRoute(readersHolderRef, nodeViewHolderRef, ergoSettings), + WalletApiRoute(readersHolderRef, nodeViewHolderRef, ergoSettings), + UtxoApiRoute(readersHolderRef, scorexSettings.restApi), + ScriptApiRoute(readersHolderRef, ergoSettings), + ScanApiRoute(readersHolderRef, ergoSettings), + NodeApiRoute(ergoSettings) + ) ++ minerRefOpt.map(minerRef => MiningApiRoute(minerRef, ergoSettings)).toSeq private val swaggerRoute = SwaggerRoute(scorexSettings.restApi, swaggerConfig) private val panelRoute = NodePanelRoute() @@ -211,7 +219,7 @@ class ErgoApp(args: Args) extends ScorexLogging { } private def swaggerConfig: String = - Source.fromResource("api/openapi.yaml").getLines.mkString("\n") + Source.fromResource("api/openapi.yaml")(Codec.UTF8).getLines.mkString("\n") private def run(): Future[ServerBinding] = { require(scorexSettings.network.agentName.length <= ErgoApp.ApplicationNameLimit) diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index 6590bce1df..cef83cbb00 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -1,10 +1,9 @@ package org.ergoplatform.http.api import java.math.BigInteger - import io.circe._ import org.bouncycastle.util.BigIntegers -import org.ergoplatform.{ErgoBox, ErgoLikeContext, ErgoLikeTransaction, JsonCodecs, UnsignedErgoLikeTransaction} +import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, ErgoLikeContext, ErgoLikeTransaction, JsonCodecs, UnsignedErgoLikeTransaction} import org.ergoplatform.http.api.ApiEncoderOption.Detalization import org.ergoplatform.ErgoBox.RegisterId import org.ergoplatform.mining.{groupElemFromBytes, groupElemToBytes} @@ -29,6 +28,8 @@ import sigmastate.interpreter._ import sigmastate.interpreter.CryptoConstants.EcPointType import io.circe.syntax._ import org.ergoplatform.http.api.requests.{CryptoResult, ExecuteRequest, HintExtractionRequest} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.getAddress +import org.ergoplatform.nodeView.history.extra.{BalanceInfo, IndexedErgoBox, IndexedErgoTransaction, IndexedToken} import org.ergoplatform.wallet.interface4j.SecretString import scorex.crypto.authds.{LeafData, Side} import scorex.crypto.authds.merkle.MerkleProof @@ -52,6 +53,8 @@ trait ApiCodecs extends JsonCodecs { implicit val sideEncoder: Encoder[Side] = _.toByte.asJson + implicit val ergoAddressEncoder: ErgoAddressEncoder = null + protected implicit def merkleProofEncoder[D <: Digest]: Encoder[MerkleProof[D]] = { proof => Json.obj( "leafData" -> proof.leafData.asJson, @@ -470,6 +473,77 @@ trait ApiCodecs extends JsonCodecs { fields.asJson } + implicit val indexedBoxEncoder: Encoder[IndexedErgoBox] = { iEb => + iEb.box.asJson.deepMerge(Json.obj( + "globalIndex" -> iEb.globalIndex.asJson, + "inclusionHeight" -> iEb.inclusionHeight.asJson, + "address" -> ergoAddressEncoder.toString(getAddress(iEb.box.ergoTree)).asJson, + "spentTransactionId" -> iEb.spendingTxIdOpt.asJson + )) + } + + implicit val indexedBoxSeqEncoder: Encoder[(Seq[IndexedErgoBox],Long)] = { iEbSeq => + Json.obj( + "items" -> iEbSeq._1.asJson, + "total" -> iEbSeq._2.asJson + ) + } + + implicit val indexedTxEncoder: Encoder[IndexedErgoTransaction] = { iEt => + Json.obj( + "id" -> iEt.txid.asJson, + "blockId" -> iEt.blockId.asJson, + "inclusionHeight" -> iEt.inclusionHeight.asJson, + "timestamp" -> iEt.timestamp.asJson, + "index" -> iEt.index.asJson, + "globalIndex" -> iEt.globalIndex.asJson, + "numConfirmations" -> iEt.numConfirmations.asJson, + "inputs" -> iEt.inputs.asJson, + "dataInputs" -> iEt.dataInputs.asJson, + "outputs" -> iEt.outputs.asJson, + "size" -> iEt.txSize.asJson + ) + } + + implicit val indexedTxSeqEncoder: Encoder[(Seq[IndexedErgoTransaction],Long)] = { iEtSeq => + Json.obj( + "items" -> iEtSeq._1.asJson, + "total" -> iEtSeq._2.asJson + ) + } + + implicit val IndexedTokenEncoder: Encoder[IndexedToken] = { token => + Json.obj( + "id" -> token.tokenId.asJson, + "boxId" -> token.boxId.asJson, + "emissionAmount" -> token.amount.asJson, + "name" -> token.name.asJson, + "description" -> token.description.asJson, + "decimals" -> token.decimals.asJson + ) + } + + implicit val BalanceInfoEncoder: Encoder[BalanceInfo] = { bal => + Json.obj( + "nanoErgs" -> bal.nanoErgs.asJson, + "tokens" -> bal.tokens.map(token => { + Json.obj( + "tokenId" -> token._1.asJson, + "amount" -> token._2.asJson, + "decimals" -> bal.additionalTokenInfo(token._1)._2.asJson, + "name" -> bal.additionalTokenInfo(token._1)._1.asJson + ) + }).asJson + ) + } + + implicit val TotalBalanceInfoEncoder: Encoder[(BalanceInfo,BalanceInfo)] = { tBal => + Json.obj( + "confirmed" -> tBal._1.asJson, + "unconfirmed" -> tBal._2.asJson + ) + } + } trait ApiEncoderOption diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala new file mode 100644 index 0000000000..3b79ea3b27 --- /dev/null +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -0,0 +1,298 @@ +package org.ergoplatform.http.api + +import akka.actor.{ActorRef, ActorRefFactory} +import akka.http.scaladsl.server.{Directive, Directive1, Route, ValidationRejection} +import akka.pattern.ask +import io.circe.Json +import io.circe.syntax.EncoderOps +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} +import org.ergoplatform.nodeView.ErgoReadersHolder.{GetDataFromHistory, GetReaders, Readers} +import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{GlobalBoxIndexKey, GlobalTxIndexKey} +import org.ergoplatform.nodeView.history.extra._ +import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader +import org.ergoplatform.settings.ErgoSettings +import scorex.core.api.http.ApiError.BadRequest +import scorex.core.api.http.ApiResponse +import scorex.core.settings.RESTApiSettings +import scorex.util.{ModifierId, bytesToId} +import sigmastate.Values.ErgoTree +import spire.implicits.cfor + +import java.nio.ByteBuffer +import scala.concurrent.Future +import scala.util.Success + +case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings) + (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute with ApiCodecs { + + val settings: RESTApiSettings = ergoSettings.scorexSettings.restApi + + private val paging: Directive[(Int, Int)] = parameters("offset".as[Int] ? 0, "limit".as[Int] ? 5) + + /** + * Total number of boxes/transactions that can be requested at once to avoid too heavy requests ([[BlocksApiRoute.MaxHeaders]]) + */ + private val MaxItems = 16384 + + override implicit val ergoAddressEncoder: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder + + private val ergoAddress: Directive1[ErgoAddress] = entity(as[String]).flatMap(handleErgoAddress) + + private def handleErgoAddress(value: String): Directive1[ErgoAddress] = { + ergoAddressEncoder.fromString(value) match { + case Success(addr) => provide(addr) + case _ => reject(ValidationRejection("Wrong address format")) + } + } + + override val route: Route = pathPrefix("blockchain") { + getIndexedHeightR ~ + getTxByIdR ~ + getTxByIndexR ~ + getTxsByAddressR ~ + getTxRangeR ~ + getBoxByIdR ~ + getBoxByIndexR ~ + getBoxesByAddressR ~ + getBoxesByAddressUnspentR ~ + getBoxRangeR ~ + getBoxesByErgoTreeR ~ + getBoxesByErgoTreeUnspentR ~ + getTokenInfoByIdR ~ + getAddressBalanceTotalR + } + + private def getHistory: Future[ErgoHistoryReader] = + (readersHolder ? GetDataFromHistory[ErgoHistoryReader](r => r)).mapTo[ErgoHistoryReader] + + private def getHistoryWithMempool: Future[(ErgoHistoryReader,ErgoMemPoolReader)] = + (readersHolder ? GetReaders).mapTo[Readers].map(r => (r.h, r.m)) + + private def getAddress(tree: ErgoTree)(history: ErgoHistoryReader): Option[IndexedErgoAddress] = { + history.typedExtraIndexById[IndexedErgoAddress](bytesToId(IndexedErgoAddressSerializer.hashErgoTree(tree))) + } + + private def getAddress(addr: ErgoAddress)(history: ErgoHistoryReader): Option[IndexedErgoAddress] = getAddress(addr.script)(history) + + private def getTxById(id: ModifierId)(history: ErgoHistoryReader): Option[IndexedErgoTransaction] = + history.typedExtraIndexById[IndexedErgoTransaction](id) match { + case Some(tx) => Some(tx.retrieveBody(history)) + case None => None + } + + private def getTxByIdF(id: ModifierId): Future[Option[IndexedErgoTransaction]] = + getHistory.map { history => + getTxById(id)(history) + } + + private def getIndexedHeightF: Future[Json] = + getHistory.map { history => + Json.obj( + "indexedHeight" -> ExtraIndexer.getIndex(ExtraIndexer.IndexedHeightKey)(history).getInt().asJson, + "fullHeight" -> history.fullBlockHeight.asJson + ) + } + + private def getIndexedHeightR: Route = (pathPrefix("indexedHeight") & get) { + ApiResponse(getIndexedHeightF) + } + + private def getTxByIdR: Route = (get & pathPrefix("transaction" / "byId") & modifierId) { id => + ApiResponse(getTxByIdF(id)) + } + + private def getTxByIndex(index: Long)(history: ErgoHistoryReader): Option[IndexedErgoTransaction] = + getTxById(history.typedExtraIndexById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(index))).get.m)(history) + + private def getLastTx(history: ErgoHistoryReader): IndexedErgoTransaction = + getTxByIndex(ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalTxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong - 1)(history).get + + private def getTxByIndexF(index: Long): Future[Option[IndexedErgoTransaction]] = + getHistory.map { history => + getTxByIndex(index)(history) + } + + private def getTxByIndexR: Route = (pathPrefix("transaction" / "byIndex" / LongNumber) & get) { index => + ApiResponse(getTxByIndexF(index)) + } + + private def getTxsByAddress(addr: ErgoAddress, offset: Int, limit: Int): Future[(Seq[IndexedErgoTransaction],Long)] = + getHistory.map { history => + getAddress(addr)(history) match { + case Some(addr) => (addr.retrieveTxs(history, offset, limit), addr.txCount()) + case None => (Seq.empty[IndexedErgoTransaction], 0L) + } + } + + private def getTxsByAddressR: Route = (post & pathPrefix("transaction" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems transactions can be requested") + }else { + ApiResponse(getTxsByAddress(address, offset, limit)) + } + } + + private def getTxRange(offset: Int, limit: Int): Future[Seq[ModifierId]] = + getHistory.map { history => + val base: Int = getLastTx(history).globalIndex.toInt - offset + val txIds: Array[ModifierId] = new Array[ModifierId](limit) + cfor(0)(_ < limit, _ + 1) { i => + txIds(i) = history.typedExtraIndexById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(base - limit + i))).get.m + } + txIds.reverse + } + + private def getTxRangeR: Route = (pathPrefix("transaction" / "range") & paging) { (offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems transactions can be requested") + }else { + ApiResponse(getTxRange(offset, limit)) + } + } + + private def getBoxById(id: ModifierId)(history: ErgoHistoryReader): Option[IndexedErgoBox] = + history.typedExtraIndexById[IndexedErgoBox](id) + + private def getBoxByIdF(id: ModifierId): Future[Option[IndexedErgoBox]] = + getHistory.map { history => + getBoxById(id)(history) + } + + private def getBoxByIdR: Route = (get & pathPrefix("box" / "byId") & modifierId) { id => + ApiResponse(getBoxByIdF(id)) + } + + private def getBoxByIndex(index: Long)(history: ErgoHistoryReader): Option[IndexedErgoBox] = + getBoxById(history.typedExtraIndexById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(index))).get.m)(history) + + private def getBoxByIndexF(index: Long): Future[Option[IndexedErgoBox]] = + getHistory.map { history => + getBoxByIndex(index)(history) + } + + private def getBoxByIndexR: Route = (pathPrefix("box" / "byIndex" / LongNumber) & get) { index => + ApiResponse(getBoxByIndexF(index)) + } + + private def getBoxesByAddress(addr: ErgoAddress, offset: Int, limit: Int): Future[(Seq[IndexedErgoBox],Long)] = + getHistory.map { history => + getAddress(addr)(history) match { + case Some(addr) => (addr.retrieveBoxes(history, offset, limit).reverse, addr.boxCount()) + case None => (Seq.empty[IndexedErgoBox], 0L) + } + } + + private def getBoxesByAddressR: Route = (post & pathPrefix("box" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + ApiResponse(getBoxesByAddress(address, offset, limit)) + } + } + + private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = + getHistory.map { history => + getAddress(addr)(history) match { + case Some(addr) => addr.retrieveUtxos(history, offset, limit).reverse + case None => Seq.empty[IndexedErgoBox] + } + } + + private def getBoxesByAddressUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + ApiResponse(getBoxesByAddressUnspent(address, offset, limit)) + } + } + + private def getLastBox(history: ErgoHistoryReader): IndexedErgoBox = + getBoxByIndex(ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalBoxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong - 1)(history).get + + private def getBoxRange(offset: Int, limit: Int): Future[Seq[ModifierId]] = + getHistory.map { history => + val base: Int = getLastBox(history).globalIndex.toInt - offset + val boxIds: Array[ModifierId] = new Array[ModifierId](limit) + cfor(0)(_ < limit, _ + 1) { i => + boxIds(i) = history.typedExtraIndexById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(base - limit + i))).get.m + } + boxIds.reverse + } + + private def getBoxRangeR: Route = (pathPrefix("box" / "range") & paging) { (offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + ApiResponse(getBoxRange(offset, limit)) + } + } + + private def getBoxesByErgoTree(tree: ErgoTree, offset: Int, limit: Int): Future[(Seq[IndexedErgoBox],Long)] = + getHistory.map { history => + getAddress(tree)(history) match { + case Some(iEa) => (iEa.retrieveBoxes(history, offset, limit).reverse, iEa.boxCount()) + case None => (Seq.empty[IndexedErgoBox], 0L) + } + } + + private def getBoxesByErgoTreeR: Route = (post & pathPrefix("box" / "byErgoTree") & ergoTree & paging) { (tree, offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + ApiResponse(getBoxesByErgoTree(tree, offset, limit)) + } + } + + private def getBoxesByErgoTreeUnspent(tree: ErgoTree, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = + getHistory.map { history => + getAddress(tree)(history) match { + case Some(iEa) => iEa.retrieveUtxos(history, offset, limit).reverse + case None => Seq.empty[IndexedErgoBox] + } + } + + private def getBoxesByErgoTreeUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byErgoTree") & ergoTree & paging) { (tree, offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + ApiResponse(getBoxesByErgoTreeUnspent(tree, offset, limit)) + } + } + + private def getTokenInfoById(id: ModifierId): Future[Option[IndexedToken]] = { + getHistory.map { history => + history.typedExtraIndexById[IndexedToken](IndexedTokenSerializer.uniqueId(id)) + } + } + + private def getTokenInfoByIdR: Route = (get & pathPrefix("token" / "byId") & modifierId) { id => + ApiResponse(getTokenInfoById(id)) + } + + private def getUnconfirmedForAddress(address: ErgoAddress)(mempool: ErgoMemPoolReader): BalanceInfo = { + val bal: BalanceInfo = new BalanceInfo + mempool.getAll.map(_.transaction).foreach(tx => { + tx.outputs.foreach(box => { + if(address.equals(ExtraIndexer.getAddress(box.ergoTree))) bal.add(box) + }) + }) + bal + } + + private def getAddressBalanceTotal(address: ErgoAddress): Future[(BalanceInfo,BalanceInfo)] = { + getHistoryWithMempool.map { case (history, mempool) => + getAddress(address)(history) match { + case Some(addr) => + (addr.balanceInfo.get.retreiveAdditionalTokenInfo(history), getUnconfirmedForAddress(address)(mempool).retreiveAdditionalTokenInfo(history)) + case None => + (new BalanceInfo, getUnconfirmedForAddress(address)(mempool).retreiveAdditionalTokenInfo(history)) + } + } + } + + private def getAddressBalanceTotalR: Route = (post & pathPrefix("balance") & ergoAddress) { address => + ApiResponse(getAddressBalanceTotal(address)) + } + +} diff --git a/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala index ea27de357a..0922a83a8f 100644 --- a/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala @@ -22,7 +22,7 @@ case class EmissionApiRoute(ergoSettings: ErgoSettings) private val reemissionSettings = chainSettings.reemission - private implicit val ergoAddressEncoder: ErgoAddressEncoder = new ErgoAddressEncoder(chainSettings.addressPrefix) + override implicit val ergoAddressEncoder: ErgoAddressEncoder = new ErgoAddressEncoder(chainSettings.addressPrefix) override def route: Route = pathPrefix("emission") { emissionAt ~ scripts diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala index 97949231c4..dafc0bf0a1 100644 --- a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala @@ -14,6 +14,9 @@ import akka.pattern.ask import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedTransaction import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome._ +import scorex.util.encode.Base16 +import sigmastate.Values.ErgoTree +import sigmastate.serialization.ErgoTreeSerializer import scala.concurrent.{ExecutionContextExecutor, Future} import scala.util.{Success, Try} @@ -34,6 +37,16 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { } } + val ergoTree: Directive1[ErgoTree] = entity(as[String]).flatMap(handleErgoTree) + + private def handleErgoTree(value: String): Directive1[ErgoTree] = { + Base16.decode(value) match { + case Success(bytes) => provide(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(bytes)) + case _ => reject(ValidationRejection("Invalid hex data")) + } + + } + private def getStateAndPool(readersHolder: ActorRef): Future[(ErgoStateReader, ErgoMemPoolReader)] = { (readersHolder ? GetReaders).mapTo[Readers].map { rs => (rs.s, rs.m) @@ -78,9 +91,9 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { val maxTxCost = ergoSettings.nodeSettings.maxTransactionCost utxo.withMempool(mp) .validateWithCost(tx, maxTxCost) - .map(cost => UnconfirmedTransaction(tx, Some(cost), now, now, bytes, source = None)) + .map(cost => new UnconfirmedTransaction(tx, Some(cost), now, now, bytes, source = None)) case _ => - tx.statelessValidity().map(_ => UnconfirmedTransaction(tx, None, now, now, bytes, source = None)) + tx.statelessValidity().map(_ => new UnconfirmedTransaction(tx, None, now, now, bytes, source = None)) } } diff --git a/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala index e583790be6..ce04c7fbe8 100644 --- a/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala @@ -7,16 +7,17 @@ import io.circe.syntax._ import org.ergoplatform.local.ErgoStatsCollector.{GetNodeInfo, NodeInfo} import scorex.core.api.http.ApiResponse import scorex.core.settings.RESTApiSettings -import scorex.core.utils.NetworkTimeProvider +/** + * API methods corresponding to /info path + */ case class InfoApiRoute(statsCollector: ActorRef, - settings: RESTApiSettings, - timeProvider: NetworkTimeProvider) + settings: RESTApiSettings) (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute { override val route: Route = (path("info") & get) { - val timeJson = Map("currentTime" -> timeProvider.time().asJson).asJson + val timeJson = Map("currentTime" -> System.currentTimeMillis().asJson).asJson ApiResponse((statsCollector ? GetNodeInfo).mapTo[NodeInfo].map(_.asJson.deepMerge(timeJson))) } diff --git a/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala b/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala index bb41748619..febc4eebb3 100644 --- a/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala +++ b/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala @@ -16,8 +16,6 @@ import scorex.core.network.ConnectedPeer import scorex.core.network.NetworkController.ReceivableMessages.{GetConnectedPeers, GetPeersStatus} import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages._ import org.ergoplatform.network.ErgoSyncTracker -import scorex.core.utils.NetworkTimeProvider -import scorex.core.utils.TimeProvider.Time import scorex.util.ScorexLogging import scorex.core.network.peer.PeersStatus @@ -31,8 +29,7 @@ import scala.concurrent.duration._ class ErgoStatsCollector(readersHolder: ActorRef, networkController: ActorRef, syncTracker: ErgoSyncTracker, - settings: ErgoSettings, - timeProvider: NetworkTimeProvider) + settings: ErgoSettings) extends Actor with ScorexLogging { override def preStart(): Unit = { @@ -47,8 +44,6 @@ class ErgoStatsCollector(readersHolder: ActorRef, context.system.scheduler.scheduleAtFixedRate(45.seconds, 30.seconds, networkController, GetPeersStatus)(ec, self) } - private def networkTime(): Time = timeProvider.time() - private var nodeInfo = NodeInfo( settings.scorexSettings.network.nodeName, Version.VersionString, @@ -64,12 +59,13 @@ class ErgoStatsCollector(readersHolder: ActorRef, None, None, None, - launchTime = networkTime(), - lastIncomingMessageTime = networkTime(), + launchTime = System.currentTimeMillis(), + lastIncomingMessageTime = System.currentTimeMillis(), None, LaunchParameters, eip27Supported = true, - settings.scorexSettings.restApi.publicUrl) + settings.scorexSettings.restApi.publicUrl, + settings.nodeSettings.extraIndex) override def receive: Receive = onConnectedPeers orElse @@ -91,7 +87,7 @@ class ErgoStatsCollector(readersHolder: ActorRef, headersScore = h.bestHeaderOpt.flatMap(m => h.scoreOf(m.id)), fullBlocksScore = h.bestFullBlockOpt.flatMap(m => h.scoreOf(m.id)), genesisBlockIdOpt = h.headerIdsAtHeight(ErgoHistory.GenesisHeight).headOption, - stateRoot = Some(Algos.encode(s.rootHash)), + stateRoot = Some(Algos.encode(s.rootDigest)), stateVersion = Some(s.version), parameters = s.stateContext.currentParameters ) @@ -171,12 +167,14 @@ object ErgoStatsCollector { * @param headersScore - cumulative difficulty of best headers-chain * @param bestFullBlockOpt - best full-block id (header id of such block) * @param fullBlocksScore - cumulative difficulty of best full blocks chain + * @param maxPeerHeight - maximum block height of connected peers * @param launchTime - when the node was launched (in Java time format, basically, UNIX time * 1000) * @param lastIncomingMessageTime - when the node received last p2p message (in Java time) * @param genesisBlockIdOpt - header id of genesis block * @param parameters - array with network parameters at the moment * @param eip27Supported - whether EIP-27 locked in - * @param restApiUrl publicly accessible url of node which exposes restApi in firewall + * @param restApiUrl - publicly accessible url of node which exposes restApi in firewall + * @param extraIndex - whether the node has additional indexing enabled */ case class NodeInfo(nodeName: String, appVersion: String, @@ -191,13 +189,14 @@ object ErgoStatsCollector { headersScore: Option[BigInt], bestFullBlockOpt: Option[ErgoFullBlock], fullBlocksScore: Option[BigInt], - maxPeerHeight : Option[Int], // Maximum block height of connected peers + maxPeerHeight : Option[Int], launchTime: Long, lastIncomingMessageTime: Long, genesisBlockIdOpt: Option[String], parameters: Parameters, eip27Supported: Boolean, - restApiUrl: Option[URL]) + restApiUrl: Option[URL], + extraIndex: Boolean) object NodeInfo extends ApiCodecs { implicit val paramsEncoder: Encoder[Parameters] = org.ergoplatform.settings.ParametersSerializer.jsonEncoder @@ -223,6 +222,7 @@ object ErgoStatsCollector { "stateType" -> ni.stateType.stateTypeName.asJson, "stateVersion" -> ni.stateVersion.asJson, "isMining" -> ni.isMining.asJson, + "isExplorer" -> ni.extraIndex.asJson, "peersCount" -> ni.peersCount.asJson, "launchTime" -> ni.launchTime.asJson, "lastSeenMessageTime" -> ni.lastIncomingMessageTime.asJson, @@ -238,19 +238,17 @@ object ErgoStatsCollector { object ErgoStatsCollectorRef { - def props(readersHolder: ActorRef, + private def props(readersHolder: ActorRef, networkController: ActorRef, syncTracker : ErgoSyncTracker, - settings: ErgoSettings, - timeProvider: NetworkTimeProvider): Props = - Props(new ErgoStatsCollector(readersHolder, networkController, syncTracker, settings, timeProvider)) + settings: ErgoSettings): Props = + Props(new ErgoStatsCollector(readersHolder, networkController, syncTracker, settings)) def apply(readersHolder: ActorRef, networkController: ActorRef, syncTracker : ErgoSyncTracker, - settings: ErgoSettings, - timeProvider: NetworkTimeProvider)(implicit system: ActorSystem): ActorRef = - system.actorOf(props(readersHolder, networkController, syncTracker, settings, timeProvider)) + settings: ErgoSettings)(implicit system: ActorSystem): ActorRef = + system.actorOf(props(readersHolder, networkController, syncTracker, settings)) } diff --git a/src/main/scala/org/ergoplatform/local/MempoolAuditor.scala b/src/main/scala/org/ergoplatform/local/MempoolAuditor.scala index c2239037c2..6578613c5e 100644 --- a/src/main/scala/org/ergoplatform/local/MempoolAuditor.scala +++ b/src/main/scala/org/ergoplatform/local/MempoolAuditor.scala @@ -4,7 +4,7 @@ import akka.actor.SupervisorStrategy.{Restart, Stop} import akka.actor.{Actor, ActorInitializationException, ActorKilledException, ActorRef, ActorRefFactory, DeathPactException, OneForOneStrategy, Props} import org.ergoplatform.local.CleanupWorker.RunCleanup import org.ergoplatform.local.MempoolAuditor.CleanupDone -import org.ergoplatform.modifiers.mempool.UnconfirmedTransaction +import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader import org.ergoplatform.settings.ErgoSettings import scorex.core.network.Broadcast @@ -12,7 +12,6 @@ import scorex.core.network.NetworkController.ReceivableMessages.SendToNetwork import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.RecheckMempool import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoStateReader} import scorex.core.network.message.{InvData, InvSpec, Message} -import scorex.core.transaction.Transaction import scorex.util.ScorexLogging import scala.concurrent.duration._ @@ -87,7 +86,7 @@ class MempoolAuditor(nodeViewHolderRef: ActorRef, private def broadcastTx(unconfirmedTx: UnconfirmedTransaction): Unit = { val msg = Message( InvSpec, - Right(InvData(Transaction.ModifierTypeId, Seq(unconfirmedTx.id))), + Right(InvData(ErgoTransaction.modifierTypeId, Seq(unconfirmedTx.id))), None ) networkControllerRef ! SendToNetwork(msg, Broadcast) diff --git a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala index a24dd82c22..dbdab5b7f2 100644 --- a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala +++ b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala @@ -24,7 +24,6 @@ import org.ergoplatform.wallet.Constants.MaxAssetsPerBox import org.ergoplatform.wallet.interpreter.ErgoInterpreter import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoScriptPredef, Input} import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{EliminateTransactions, LocallyGeneratedModifier} -import scorex.core.utils.NetworkTimeProvider import scorex.crypto.hash.Digest32 import scorex.util.encode.Base16 import scorex.util.{ModifierId, ScorexLogging} @@ -46,7 +45,6 @@ class CandidateGenerator( minerPk: ProveDlog, readersHolderRef: ActorRef, viewHolderRef: ActorRef, - timeProvider: NetworkTimeProvider, ergoSettings: ErgoSettings ) extends Actor with ScorexLogging { @@ -147,7 +145,6 @@ class CandidateGenerator( state.hr, state.sr, state.mpr, - timeProvider, minerPk, txsToInclude, ergoSettings @@ -256,7 +253,6 @@ object CandidateGenerator extends ScorexLogging { minerPk: ProveDlog, readersHolderRef: ActorRef, viewHolderRef: ActorRef, - timeProvider: NetworkTimeProvider, ergoSettings: ErgoSettings )(implicit context: ActorRefFactory): ActorRef = context.actorOf( @@ -265,7 +261,6 @@ object CandidateGenerator extends ScorexLogging { minerPk, readersHolderRef, viewHolderRef, - timeProvider, ergoSettings ) ).withDispatcher("critical-dispatcher"), @@ -328,7 +323,6 @@ object CandidateGenerator extends ScorexLogging { h: ErgoHistoryReader, s: UtxoStateReader, m: ErgoMemPoolReader, - timeProvider: NetworkTimeProvider, pk: ProveDlog, txsToInclude: Seq[ErgoTransaction], ergoSettings: ErgoSettings @@ -368,7 +362,6 @@ object CandidateGenerator extends ScorexLogging { h, desiredUpdate, s, - timeProvider, poolTransactions, emissionTxOpt, unspentTxsToInclude, @@ -445,7 +438,6 @@ object CandidateGenerator extends ScorexLogging { history: ErgoHistoryReader, proposedUpdate: ErgoValidationSettingsUpdate, state: UtxoStateReader, - timeProvider: NetworkTimeProvider, poolTxs: Seq[UnconfirmedTransaction], emissionTxOpt: Option[ErgoTransaction], prioritizedTransactions: Seq[ErgoTransaction], @@ -461,7 +453,7 @@ object CandidateGenerator extends ScorexLogging { // Make progress in time since last block. // If no progress is made, then, by consensus rules, the block will be rejected. val timestamp = - Math.max(timeProvider.time(), bestHeaderOpt.map(_.timestamp + 1).getOrElse(0L)) + Math.max(System.currentTimeMillis(), bestHeaderOpt.map(_.timestamp + 1).getOrElse(0L)) val stateContext = state.stateContext diff --git a/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala b/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala index fef6d1eb0f..887845cd20 100644 --- a/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala +++ b/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala @@ -9,7 +9,6 @@ import org.ergoplatform.nodeView.wallet.ErgoWalletActor.{FirstSecretResponse, Ge import org.ergoplatform.settings.ErgoSettings import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.GetDataFromCurrentView import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.FullBlockApplied -import scorex.core.utils.NetworkTimeProvider import scorex.util.ScorexLogging import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog} @@ -24,7 +23,6 @@ class ErgoMiner( ergoSettings: ErgoSettings, viewHolderRef: ActorRef, readersHolderRef: ActorRef, - timeProvider: NetworkTimeProvider, secretKeyOpt: Option[DLogProverInput] ) extends Actor with Stash @@ -62,7 +60,6 @@ class ErgoMiner( publicKey, readersHolderRef, viewHolderRef, - timeProvider, ergoSettings ) context.become(starting(MinerState(secretKeyOpt, publicKey, candidateGeneratorRef))) @@ -116,7 +113,7 @@ class ErgoMiner( /** We need to ignore all historical blocks, mining is triggered by latest blocks only */ private def shouldStartMine(b: Header): Boolean = { - b.isNew(timeProvider, ergoSettings.chainSettings.blockInterval * 2) + b.isNew(ergoSettings.chainSettings.blockInterval * 2) } /** Let's wait for a signal to start mining, either from ErgoApp or when a latest blocks get applied to blockchain */ @@ -201,12 +198,11 @@ object ErgoMiner extends ScorexLogging { ergoSettings: ErgoSettings, viewHolderRef: ActorRef, readersHolderRef: ActorRef, - timeProvider: NetworkTimeProvider, skOpt: Option[DLogProverInput] = None )(implicit context: ActorRefFactory): ActorRef = context.actorOf( Props( - new ErgoMiner(ergoSettings, viewHolderRef, readersHolderRef, timeProvider, skOpt) + new ErgoMiner(ergoSettings, viewHolderRef, readersHolderRef, skOpt) ) ) } diff --git a/src/main/scala/org/ergoplatform/modifiers/ErgoFullBlock.scala b/src/main/scala/org/ergoplatform/modifiers/ErgoFullBlock.scala index 80240799d0..6fb1a0ff43 100644 --- a/src/main/scala/org/ergoplatform/modifiers/ErgoFullBlock.scala +++ b/src/main/scala/org/ergoplatform/modifiers/ErgoFullBlock.scala @@ -8,7 +8,7 @@ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.history.{ADProofs, BlockTransactions} import org.ergoplatform.modifiers.mempool.ErgoTransaction import scorex.core.serialization.ScorexSerializer -import scorex.core.{ModifierTypeId, TransactionsCarryingPersistentNodeViewModifier} +import scorex.core.TransactionsCarryingPersistentNodeViewModifier import scorex.util.ModifierId case class ErgoFullBlock(header: Header, @@ -20,7 +20,7 @@ case class ErgoFullBlock(header: Header, override type M = ErgoFullBlock - override val modifierTypeId: ModifierTypeId = ErgoFullBlock.modifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = ErgoFullBlock.modifierTypeId override def serializedId: Array[Byte] = header.serializedId @@ -49,7 +49,7 @@ case class ErgoFullBlock(header: Header, object ErgoFullBlock extends ApiCodecs { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (-127: Byte) + val modifierTypeId: NetworkObjectTypeId.Value = FullBlockTypeId.value implicit val jsonEncoder: Encoder[ErgoFullBlock] = { b: ErgoFullBlock => Json.obj( diff --git a/src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala b/src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala new file mode 100644 index 0000000000..8e258d96d6 --- /dev/null +++ b/src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala @@ -0,0 +1,70 @@ +package org.ergoplatform.modifiers + +import supertagged.TaggedType +import NetworkObjectTypeId._ + +/** + * Hierarchy encoding blockchain objects being sent over wire: block sections, chunks of UTXO set snapshot etc + */ +sealed trait NetworkObjectTypeId { + /** + * 1-byte ID of network object type + */ + val value: NetworkObjectTypeId.Value +} + +object NetworkObjectTypeId { + object Value extends TaggedType[Byte] + type Value = Value.Type + + @inline + def fromByte(value: Byte): Value = Value @@ value +} + +/** + * Unconfirmed transactions sent outside blocks + */ +object TransactionTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(2) +} + +/** + * Block header, section of a block PoW is done on top of. This section is committing to other sections + */ +object HeaderTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(101) +} + +/** + * Block transactions sections. Contains all the transactions for a block. + */ +object BlockTransactionsTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(102) +} + +/** + * Block section which contains proofs of correctness for UTXO set transformations. + * The section contains proofs for all the transformations (i.e. for all the block transactions) + */ +object ProofsTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(104) +} + + +/** + * Block section which contains key-value pairs with different additional data. + * Interlinks vector (for nipopow proofs) written there, as well as current network parameters + * (at the beginning of voting epoch), but miners can also put arbitrary data there. + */ +object ExtensionTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(108) +} + +/** + * Virtual object which is not being sent over the wire rather, constructed locally from different sections + * got over the wire (header, transactions, extension in the "utxo" mode, those three sections plus proofs in + * the "digest" mode). + */ +object FullBlockTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(-127) +} diff --git a/src/main/scala/org/ergoplatform/modifiers/NonHeaderBlockSection.scala b/src/main/scala/org/ergoplatform/modifiers/NonHeaderBlockSection.scala index bbed96e1ff..5a18374056 100644 --- a/src/main/scala/org/ergoplatform/modifiers/NonHeaderBlockSection.scala +++ b/src/main/scala/org/ergoplatform/modifiers/NonHeaderBlockSection.scala @@ -1,7 +1,6 @@ package org.ergoplatform.modifiers import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId import scorex.crypto.hash.Digest32 import scorex.util.{ModifierId, bytesToId, idToBytes} @@ -10,7 +9,8 @@ import scorex.util.{ModifierId, bytesToId, idToBytes} */ trait NonHeaderBlockSection extends BlockSection { - override lazy val serializedId: Array[Byte] = NonHeaderBlockSection.computeIdBytes(modifierTypeId, headerId, digest) + override lazy val serializedId: Array[Byte] = + NonHeaderBlockSection.computeIdBytes(modifierTypeId, headerId, digest) override lazy val id: ModifierId = bytesToId(serializedId) @@ -22,9 +22,9 @@ trait NonHeaderBlockSection extends BlockSection { } object NonHeaderBlockSection { - def computeId(modifierType: ModifierTypeId, headerId: ModifierId, digest: Array[Byte]): ModifierId = + def computeId(modifierType: NetworkObjectTypeId.Value, headerId: ModifierId, digest: Array[Byte]): ModifierId = bytesToId(computeIdBytes(modifierType, headerId, digest)) - def computeIdBytes(modifierType: ModifierTypeId, headerId: ModifierId, digest: Array[Byte]): Array[Byte] = + def computeIdBytes(modifierType: NetworkObjectTypeId.Value, headerId: ModifierId, digest: Array[Byte]): Array[Byte] = Algos.hash.prefixedHash(modifierType, idToBytes(headerId), digest) } diff --git a/src/main/scala/org/ergoplatform/modifiers/history/ADProofs.scala b/src/main/scala/org/ergoplatform/modifiers/history/ADProofs.scala index 4471c7da30..dd3dc69966 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/ADProofs.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/ADProofs.scala @@ -3,11 +3,10 @@ package org.ergoplatform.modifiers.history import io.circe.syntax._ import io.circe.{Decoder, Encoder, HCursor} import org.ergoplatform.http.api.ApiCodecs -import org.ergoplatform.modifiers.NonHeaderBlockSection +import org.ergoplatform.modifiers.{NetworkObjectTypeId, NonHeaderBlockSection, ProofsTypeId} import org.ergoplatform.modifiers.state._ import org.ergoplatform.settings.Algos.HF import org.ergoplatform.settings.{Algos, Constants} -import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.crypto.authds.avltree.batch.{Lookup => _, _} import scorex.crypto.authds.{ADDigest, ADValue, SerializedAdProof} @@ -24,7 +23,7 @@ case class ADProofs(headerId: ModifierId, override def digest: Digest32 = ADProofs.proofDigest(proofBytes) - override val modifierTypeId: ModifierTypeId = ADProofs.modifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = ADProofs.modifierTypeId override type M = ADProofs @@ -69,7 +68,8 @@ case class ADProofs(headerId: ModifierId, } object ADProofs extends ApiCodecs { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (104: Byte) + + val modifierTypeId: NetworkObjectTypeId.Value = ProofsTypeId.value val KL = 32 diff --git a/src/main/scala/org/ergoplatform/modifiers/history/BlockTransactions.scala b/src/main/scala/org/ergoplatform/modifiers/history/BlockTransactions.scala index cb926f3894..12ce33b24d 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/BlockTransactions.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/BlockTransactions.scala @@ -3,7 +3,7 @@ package org.ergoplatform.modifiers.history import io.circe.syntax._ import io.circe.{Decoder, Encoder, HCursor} import org.ergoplatform.http.api.ApiCodecs -import org.ergoplatform.modifiers.NonHeaderBlockSection +import org.ergoplatform.modifiers.{BlockTransactionsTypeId, NetworkObjectTypeId, NonHeaderBlockSection} import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.{ErgoTransaction, ErgoTransactionSerializer} import org.ergoplatform.nodeView.mempool.TransactionMembershipProof @@ -37,7 +37,7 @@ case class BlockTransactions(headerId: ModifierId, assert(txs.nonEmpty, "Block should always contain at least 1 coinbase-like transaction") - override val modifierTypeId: ModifierTypeId = BlockTransactions.modifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = BlockTransactions.modifierTypeId /** * Ids of block transactions @@ -94,7 +94,7 @@ case class BlockTransactions(headerId: ModifierId, object BlockTransactions extends ApiCodecs { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (102: Byte) + val modifierTypeId: NetworkObjectTypeId.Value = BlockTransactionsTypeId.value // Used in the miner when a BlockTransaction instance is not generated yet (because a header is not known) def transactionsRoot(txs: Seq[ErgoTransaction], blockVersion: Version): Digest32 = { diff --git a/src/main/scala/org/ergoplatform/modifiers/history/extension/Extension.scala b/src/main/scala/org/ergoplatform/modifiers/history/extension/Extension.scala index 38667c2ebd..2b5e03b2f8 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/extension/Extension.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/extension/Extension.scala @@ -4,9 +4,8 @@ import com.google.common.primitives.Bytes import io.circe.syntax._ import io.circe.{Decoder, Encoder, HCursor} import org.ergoplatform.http.api.ApiCodecs -import org.ergoplatform.modifiers.NonHeaderBlockSection +import org.ergoplatform.modifiers.{ExtensionTypeId, NetworkObjectTypeId, NonHeaderBlockSection} import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.crypto.authds.LeafData import scorex.crypto.authds.merkle.MerkleTree @@ -25,7 +24,7 @@ case class Extension(headerId: ModifierId, override val sizeOpt: Option[Int] = None) extends ExtensionCandidate(fields) with NonHeaderBlockSection { - override val modifierTypeId: ModifierTypeId = Extension.modifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = Extension.modifierTypeId override type M = Extension @@ -55,7 +54,7 @@ object Extension extends ApiCodecs { Algos.merkleTree(LeafData @@ fields.map(kvToLeaf)) } - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (108: Byte) + val modifierTypeId: NetworkObjectTypeId.Value = ExtensionTypeId.value implicit val jsonEncoder: Encoder[Extension] = { e: Extension => Map( diff --git a/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala b/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala index 58919ee96d..5523fa4b85 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala @@ -7,14 +7,12 @@ import org.ergoplatform.mining.difficulty.RequiredDifficulty import org.ergoplatform.mining.AutolykosSolution import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.{ADProofs, BlockTransactions, PreHeader} -import org.ergoplatform.modifiers.{NonHeaderBlockSection, BlockSection} +import org.ergoplatform.modifiers.{BlockSection, HeaderTypeId, NetworkObjectTypeId, NonHeaderBlockSection} import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.history.ErgoHistory.Difficulty import org.ergoplatform.settings.{Algos, Constants} import org.ergoplatform.wallet.interpreter.ErgoInterpreter import scorex.core.serialization.ScorexSerializer -import scorex.core.utils.NetworkTimeProvider -import scorex.core.ModifierTypeId import scorex.crypto.authds.ADDigest import scorex.crypto.hash.Digest32 import scorex.util._ @@ -60,7 +58,7 @@ case class Header(override val version: Header.Version, override type M = Header - override val modifierTypeId: ModifierTypeId = Header.modifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = Header.modifierTypeId lazy val requiredDifficulty: Difficulty = RequiredDifficulty.decodeCompactBits(nBits) @@ -75,7 +73,7 @@ case class Header(override val version: Header.Version, /** * Expected identifiers of the block sections corresponding to this header */ - lazy val sectionIds: Seq[(ModifierTypeId, ModifierId)] = + lazy val sectionIds: Seq[(NetworkObjectTypeId.Value, ModifierId)] = Array( (ADProofs.modifierTypeId, ADProofsId), (BlockTransactions.modifierTypeId, transactionsId), @@ -86,7 +84,7 @@ case class Header(override val version: Header.Version, * Expected identifiers of the block sections corresponding to this header, * except of state transformations proof section id */ - lazy val sectionIdsWithNoProof: Seq[(ModifierTypeId, ModifierId)] = sectionIds.tail + lazy val sectionIdsWithNoProof: Seq[(NetworkObjectTypeId.Value, ModifierId)] = sectionIds.tail override lazy val toString: String = s"Header(${this.asJson.noSpaces})" @@ -102,8 +100,8 @@ case class Header(override val version: Header.Version, /** * Estimate that this header is recent enough to possibly be the best header */ - def isNew(timeProvider: NetworkTimeProvider, timeDiff: FiniteDuration): Boolean = { - timeProvider.time() - timestamp < timeDiff.toMillis + def isNew(timeDiff: FiniteDuration): Boolean = { + System.currentTimeMillis() - timestamp < timeDiff.toMillis } /** @@ -157,7 +155,7 @@ object Header extends ApiCodecs { votes = header.votes.toColl ) - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (101: Byte) + val modifierTypeId: NetworkObjectTypeId.Value = HeaderTypeId.value lazy val GenesisParentId: ModifierId = bytesToId(Array.fill(Constants.HashLength)(0: Byte)) diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala index bbf834acf1..983205c8ed 100644 --- a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala @@ -6,7 +6,7 @@ import org.ergoplatform.SigmaConstants.{MaxBoxSize, MaxPropositionBytes} import org.ergoplatform._ import org.ergoplatform.http.api.ApiCodecs import org.ergoplatform.mining.emission.EmissionRules -import org.ergoplatform.modifiers.ErgoNodeViewModifier +import org.ergoplatform.modifiers.{ErgoNodeViewModifier, NetworkObjectTypeId, TransactionTypeId} import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.ErgoTransaction.unresolvedIndices import org.ergoplatform.nodeView.ErgoContext @@ -474,6 +474,8 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input], object ErgoTransaction extends ApiCodecs with ScorexLogging with ScorexEncoding { + val modifierTypeId: NetworkObjectTypeId.Value = TransactionTypeId.value + def apply(inputs: IndexedSeq[Input], outputCandidates: IndexedSeq[ErgoBoxCandidate]): ErgoTransaction = ErgoTransaction(inputs, IndexedSeq.empty, outputCandidates, None) diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala index 5c99a0e4e3..bd18f338d8 100644 --- a/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala +++ b/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala @@ -13,12 +13,12 @@ import scorex.util.{ModifierId, ScorexLogging} * @param transactionBytes - transaction bytes, to avoid serializations when we send it over the wire * @param source - peer which delivered the transaction (None if transaction submitted via API) */ -case class UnconfirmedTransaction(transaction: ErgoTransaction, - lastCost: Option[Int], - createdTime: Long, - lastCheckedTime: Long, - transactionBytes: Option[Array[Byte]], - source: Option[ConnectedPeer]) +class UnconfirmedTransaction(val transaction: ErgoTransaction, + val lastCost: Option[Int], + val createdTime: Long, + val lastCheckedTime: Long, + val transactionBytes: Option[Array[Byte]], + val source: Option[ConnectedPeer]) extends ScorexLogging { def id: ModifierId = transaction.id @@ -27,7 +27,13 @@ case class UnconfirmedTransaction(transaction: ErgoTransaction, * Updates cost and last checked time of unconfirmed transaction */ def withCost(cost: Int): UnconfirmedTransaction = { - copy(lastCost = Some(cost), lastCheckedTime = System.currentTimeMillis()) + new UnconfirmedTransaction( + transaction, + lastCost = Some(cost), + createdTime, + lastCheckedTime = System.currentTimeMillis(), + transactionBytes, + source) } override def equals(obj: Any): Boolean = obj match { @@ -42,12 +48,12 @@ object UnconfirmedTransaction { def apply(tx: ErgoTransaction, source: Option[ConnectedPeer]): UnconfirmedTransaction = { val now = System.currentTimeMillis() - UnconfirmedTransaction(tx, None, now, now, Some(tx.bytes), source) + new UnconfirmedTransaction(tx, None, now, now, Some(tx.bytes), source) } def apply(tx: ErgoTransaction, txBytes: Array[Byte], source: Option[ConnectedPeer]): UnconfirmedTransaction = { val now = System.currentTimeMillis() - UnconfirmedTransaction(tx, None, now, now, Some(txBytes), source) + new UnconfirmedTransaction(tx, None, now, now, Some(txBytes), source) } } diff --git a/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotChunk.scala b/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotChunk.scala deleted file mode 100644 index caf93c3afd..0000000000 --- a/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotChunk.scala +++ /dev/null @@ -1,39 +0,0 @@ -package org.ergoplatform.modifiers.state - -import org.ergoplatform.ErgoBox -import org.ergoplatform.modifiers.BlockSection -import org.ergoplatform.modifiers.state.UTXOSnapshotChunk.StateElement -import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId -import scorex.core.serialization.ScorexSerializer -import scorex.crypto.authds.LeafData -import scorex.crypto.hash.Digest32 -import scorex.util.{ModifierId, bytesToId} -import scorex.utils.Random - -case class UTXOSnapshotChunk(stateElements: Seq[StateElement], - index: Short) extends BlockSection { - override val modifierTypeId: ModifierTypeId = UTXOSnapshotChunk.modifierTypeId - - //TODO implement correctly - override lazy val id: ModifierId = bytesToId(Random.randomBytes(32)) - - override def parentId: ModifierId = ??? - - override def serializedId: Array[Byte] = ??? - - override type M = UTXOSnapshotChunk - - override def serializer: ScorexSerializer[UTXOSnapshotChunk] = ??? - - lazy val rootHash: Digest32 = Algos.merkleTreeRoot(stateElements.map(LeafData @@ _.bytes)) - - override val sizeOpt: Option[Int] = None - -} - -object UTXOSnapshotChunk { - type StateElement = ErgoBox - - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (107: Byte) -} diff --git a/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotManifest.scala b/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotManifest.scala deleted file mode 100644 index 2730587874..0000000000 --- a/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotManifest.scala +++ /dev/null @@ -1,42 +0,0 @@ -package org.ergoplatform.modifiers.state - -import org.ergoplatform.modifiers.BlockSection -import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId -import scorex.core.serialization.ScorexSerializer -import scorex.core.utils.concatBytes -import scorex.crypto.authds.LeafData -import scorex.crypto.hash.Digest32 -import scorex.util.{ModifierId, bytesToId, idToBytes} - -import scala.util.Try - -case class UTXOSnapshotManifest(chunkRootHashes: Seq[Array[Byte]], blockId: ModifierId) extends BlockSection { - override val modifierTypeId: ModifierTypeId = UTXOSnapshotManifest.modifierTypeId - - override def serializedId: Array[Byte] = Algos.hash(concatBytes(chunkRootHashes :+ idToBytes(blockId))) - - override lazy val id: ModifierId = bytesToId(serializedId) - - override type M = UTXOSnapshotManifest - - override def serializer: ScorexSerializer[UTXOSnapshotManifest] = ??? - - override def parentId: ModifierId = ??? - - lazy val rootHash: Digest32 = Algos.merkleTreeRoot(LeafData @@ chunkRootHashes) - - override val sizeOpt: Option[Int] = None -} - -object UTXOSnapshotManifest { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (106: Byte) - - def validate(manifest: UTXOSnapshotManifest, header: Header): Try[Unit] = Try { - require(manifest.blockId == header.id) - require(java.util.Arrays.equals(manifest.rootHash, header.stateRoot)) - ??? - } -} - diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 7a94a60ec5..dff3b03c30 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -3,8 +3,8 @@ package org.ergoplatform.network import akka.actor.SupervisorStrategy.{Restart, Stop} import akka.actor.{Actor, ActorInitializationException, ActorKilledException, ActorRef, ActorRefFactory, DeathPactException, OneForOneStrategy, Props} import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.modifiers.mempool.{ErgoTransactionSerializer, UnconfirmedTransaction} -import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.modifiers.mempool.{ErgoTransaction, ErgoTransactionSerializer, UnconfirmedTransaction} +import org.ergoplatform.modifiers.{BlockSection, NetworkObjectTypeId} import org.ergoplatform.nodeView.history.{ErgoSyncInfoV1, ErgoSyncInfoV2} import org.ergoplatform.nodeView.history._ import ErgoNodeViewSynchronizer.{CheckModifiersToDownload, IncomingTxInfo, TransactionProcessingCacheRecord} @@ -16,7 +16,7 @@ import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages._ import org.ergoplatform.nodeView.ErgoNodeViewHolder._ import scorex.core.consensus.{Equal, Fork, Nonsense, Older, Unknown, Younger} import scorex.core.network.ModifiersStatus.Requested -import scorex.core.{ModifierTypeId, NodeViewModifier, idsToString} +import scorex.core.{NodeViewModifier, idsToString} import scorex.core.network.NetworkController.ReceivableMessages.{PenalizePeer, SendToNetwork} import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages._ import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoStateReader} @@ -26,8 +26,7 @@ import scorex.core.network._ import scorex.core.network.message.{InvData, Message, ModifiersData} import scorex.core.serialization.ScorexSerializer import scorex.core.settings.NetworkSettings -import scorex.core.transaction.Transaction -import scorex.core.utils.{NetworkTimeProvider, ScorexEncoding} +import scorex.core.utils.ScorexEncoding import scorex.core.validation.MalformedModifierError import scorex.util.{ModifierId, ScorexLogging} import scorex.core.network.DeliveryTracker @@ -49,7 +48,6 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, - timeProvider: NetworkTimeProvider, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker )(implicit ex: ExecutionContext) @@ -257,7 +255,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, context.system.scheduler.scheduleAtFixedRate(healthCheckDelay, healthCheckRate, viewHolderRef, IsChainHealthy)(ex, self) } - protected def broadcastModifierInv(modTypeId: ModifierTypeId, modId: ModifierId): Unit = { + protected def broadcastModifierInv(modTypeId: NetworkObjectTypeId.Value, modId: ModifierId): Unit = { val msg = Message(InvSpec, Right(InvData(modTypeId, Seq(modId))), None) networkControllerRef ! SendToNetwork(msg, Broadcast) } @@ -271,7 +269,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * (in history database available via `historyReader` interface, or delivery tracker cache, thus * downloading of the modifier is needed. */ - private def downloadRequired(historyReader: ErgoHistory)(modifierTypeId: ModifierTypeId, id: ModifierId): Boolean = { + private def downloadRequired(historyReader: ErgoHistory)(modifierTypeId: NetworkObjectTypeId.Value, id: ModifierId): Boolean = { deliveryTracker.status(id, modifierTypeId, Array(historyReader)) == ModifiersStatus.Unknown } @@ -341,7 +339,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, // Send history extension to the (less developed) peer 'remote' which does not have it. def sendExtension(remote: ConnectedPeer, - ext: Seq[(ModifierTypeId, ModifierId)]): Unit = { + ext: Seq[(NetworkObjectTypeId.Value, ModifierId)]): Unit = { ext.groupBy(_._1).mapValues(_.map(_._2)).foreach { case (mid, mods) => networkControllerRef ! SendToNetwork(Message(InvSpec, Right(InvData(mid, mods)), None), SendToPeer(remote)) @@ -526,7 +524,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * (non-zero if we're re-requesting the block section, in this case, there should be only * one id to request in `modifierIds` */ - def requestBlockSection(modifierTypeId: ModifierTypeId, + def requestBlockSection(modifierTypeId: NetworkObjectTypeId.Value, modifierIds: Seq[ModifierId], peer: ConnectedPeer, checksDone: Int = 0): Unit = { @@ -546,7 +544,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } def onDownloadRequest(historyReader: ErgoHistory): Receive = { - case DownloadRequest(modifiersToFetch: Map[ModifierTypeId, Seq[ModifierId]]) => + case DownloadRequest(modifiersToFetch: Map[NetworkObjectTypeId.Value, Seq[ModifierId]]) => log.debug(s"Downloading via DownloadRequest: $modifiersToFetch") if(modifiersToFetch.nonEmpty) { requestDownload( @@ -581,7 +579,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, */ protected def requestDownload(maxModifiers: Int, minModifiersPerBucket: Int, maxModifiersPerBucket: Int) (getPeersOpt: => Option[Iterable[ConnectedPeer]]) - (fetchMax: Int => Map[ModifierTypeId, Seq[ModifierId]]): Unit = + (fetchMax: Int => Map[NetworkObjectTypeId.Value, Seq[ModifierId]]): Unit = getPeersOpt .foreach { peers => val modifiersByBucket = ElementPartitioner.distribute(peers, maxModifiers, minModifiersPerBucket, maxModifiersPerBucket)(fetchMax) @@ -624,7 +622,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } private def blockSectionsFromRemote(hr: ErgoHistory, - typeId: ModifierTypeId, + typeId: NetworkObjectTypeId.Value, requestedModifiers: Map[ModifierId, Array[Byte]], remote: ConnectedPeer): Unit = { Constants.modifierSerializers.get(typeId) match { @@ -674,7 +672,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, // filter out non-requested modifiers val requestedModifiers = processSpam(remote, typeId, modifiers, blockAppliedTxsCache) - if (typeId == Transaction.ModifierTypeId) { + if (typeId == ErgoTransaction.modifierTypeId) { transactionsFromRemote(requestedModifiers, mp, remote) } else { blockSectionsFromRemote(hr, typeId, requestedModifiers, remote) @@ -687,7 +685,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, */ def parseAndProcessTransaction(id: ModifierId, bytes: Array[Byte], remote: ConnectedPeer): Unit = { if (bytes.length > settings.nodeSettings.maxTransactionSize) { - deliveryTracker.setInvalid(id, Transaction.ModifierTypeId) + deliveryTracker.setInvalid(id, ErgoTransaction.modifierTypeId) penalizeMisbehavingPeer(remote) log.warn(s"Transaction size ${bytes.length} from ${remote.toString} " + s"exceeds limit ${settings.nodeSettings.maxTransactionSize}") @@ -711,7 +709,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * @return collection of parsed modifiers */ def parseModifiers[M <: NodeViewModifier](modifiers: Map[ModifierId, Array[Byte]], - modifierTypeId: ModifierTypeId, + modifierTypeId: NetworkObjectTypeId.Value, serializer: ScorexSerializer[M], remote: ConnectedPeer): Iterable[M] = { modifiers.flatMap { case (id, bytes) => @@ -736,7 +734,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * @return ids and bytes of modifiers that were requested by our node */ def processSpam(remote: ConnectedPeer, - typeId: ModifierTypeId, + typeId: NetworkObjectTypeId.Value, modifiers: Map[ModifierId, Array[Byte]], blockAppliedTxsCache: FixedSizeApproximateCacheQueue): Map[ModifierId, Array[Byte]] = { val modifiersByStatus = @@ -747,7 +745,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, val spam = modifiersByStatus.filterKeys(_ != Requested) if (spam.nonEmpty) { - if (typeId == Transaction.ModifierTypeId) { + if (typeId == ErgoTransaction.modifierTypeId) { // penalize a peer for sending TXs that have been already applied to a block val spammyTxs = modifiers.filterKeys(blockAppliedTxsCache.mightContain) if (spammyTxs.nonEmpty) { @@ -792,7 +790,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, val modifierTypeId = invData.typeId val newModifierIds = modifierTypeId match { - case Transaction.ModifierTypeId => + case ErgoTransaction.modifierTypeId => if (txAcceptanceFilter) { val unknownMods = { @@ -847,11 +845,11 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, //other node asking for objects by their ids protected def modifiersReq(hr: ErgoHistory, mp: ErgoMemPool, invData: InvData, remote: ConnectedPeer): Unit = { val objs: Seq[(ModifierId, Array[Byte])] = invData.typeId match { - case typeId: ModifierTypeId if typeId == Transaction.ModifierTypeId => + case typeId: NetworkObjectTypeId.Value if typeId == ErgoTransaction.modifierTypeId => mp.getAll(invData.ids).map { unconfirmedTx => unconfirmedTx.transaction.id -> unconfirmedTx.transactionBytes.getOrElse(unconfirmedTx.transaction.bytes) } - case expectedTypeId: ModifierTypeId => + case expectedTypeId: NetworkObjectTypeId.Value => invData.ids.flatMap { id => hr.modifierTypeAndBytesById(id).flatMap { case (mTypeId, bytes) => if (mTypeId == expectedTypeId) { @@ -920,7 +918,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, if (deliveryTracker.status(modifierId, modifierTypeId, Seq.empty) == ModifiersStatus.Requested) { // If transaction not delivered on time, we just forget about it. // It could be removed from other peer's mempool, so no reason to penalize the peer. - if (modifierTypeId == Transaction.ModifierTypeId) { + if (modifierTypeId == ErgoTransaction.modifierTypeId) { deliveryTracker.clearStatusForModifier(modifierId, modifierTypeId, ModifiersStatus.Requested) } else { // A block section is not delivered on time. @@ -1017,7 +1015,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } } - protected def viewHolderEvents(historyReader: ErgoHistory, + private def viewHolderEvents(historyReader: ErgoHistory, mempoolReader: ErgoMemPool, blockAppliedTxsCache: FixedSizeApproximateCacheQueue): Receive = { // Requests BlockSections with `Unknown` status that are defined by block headers but not downloaded yet. @@ -1038,7 +1036,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, // If new enough semantically valid ErgoFullBlock was applied, send inv for block header and all its sections case FullBlockApplied(header) => - if (header.isNew(timeProvider, 2.hours)) { + if (header.isNew(2.hours)) { broadcastModifierInv(Header.modifierTypeId, header.id) header.sectionIds.foreach { case (mtId, id) => broadcastModifierInv(mtId, id) } } @@ -1049,7 +1047,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, case st@SuccessfulTransaction(utx) => val tx = utx.transaction - deliveryTracker.setHeld(tx.id, Transaction.ModifierTypeId) + deliveryTracker.setHeld(tx.id, ErgoTransaction.modifierTypeId) processMempoolResult(st) broadcastModifierInv(tx) @@ -1184,25 +1182,23 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, object ErgoNodeViewSynchronizer { - def props(networkControllerRef: ActorRef, + private def props(networkControllerRef: ActorRef, viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, - timeProvider: NetworkTimeProvider, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker) (implicit ex: ExecutionContext): Props = - Props(new ErgoNodeViewSynchronizer(networkControllerRef, viewHolderRef, syncInfoSpec, settings, - timeProvider, syncTracker, deliveryTracker)) + Props(new ErgoNodeViewSynchronizer(networkControllerRef, viewHolderRef, syncInfoSpec, + settings, syncTracker, deliveryTracker)) def make(viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, - timeProvider: NetworkTimeProvider, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker) (implicit context: ActorRefFactory, ex: ExecutionContext): ActorRef => ActorRef = - networkControllerRef => context.actorOf(props(networkControllerRef, viewHolderRef, syncInfoSpec, settings, timeProvider, syncTracker, deliveryTracker)) + networkControllerRef => context.actorOf(props(networkControllerRef, viewHolderRef, syncInfoSpec, settings, syncTracker, deliveryTracker)) /** * Container for aggregated costs of accepted, declined or invalidated transactions. Can be used to track global @@ -1228,8 +1224,6 @@ object ErgoNodeViewSynchronizer { // getLocalSyncInfo messages case object SendLocalSyncInfo - case class ResponseFromLocal(source: ConnectedPeer, modifierTypeId: ModifierTypeId, localObjects: Seq[(ModifierId, Array[Byte])]) - /** * Check delivery of modifier with type `modifierTypeId` and id `modifierId`. * `source` may be defined if we expect modifier from concrete peer or None if @@ -1237,7 +1231,7 @@ object ErgoNodeViewSynchronizer { * */ case class CheckDelivery(source: ConnectedPeer, - modifierTypeId: ModifierTypeId, + modifierTypeId: NetworkObjectTypeId.Value, modifierId: ModifierId) trait PeerManagerEvent @@ -1258,7 +1252,11 @@ object ErgoNodeViewSynchronizer { case class ChangedState(reader: ErgoStateReader) extends NodeViewChange - //todo: consider sending info on the rollback + /** + * Event which is published when rollback happened (on finding a better chain) + * @param branchPoint - block id which is last in the chain after rollback (before applying blocks from a fork) + */ + case class Rollback(branchPoint: ModifierId) extends NodeViewHolderEvent case object RollbackFailed extends NodeViewHolderEvent @@ -1284,19 +1282,25 @@ object ErgoNodeViewSynchronizer { */ case class FailedOnRecheckTransaction(id : ModifierId, error: Throwable) extends ModificationOutcome - case class RecoverableFailedModification(typeId: ModifierTypeId, modifierId: ModifierId, error: Throwable) extends ModificationOutcome + /** + * A signal that block section with id `modifierId` was invalidated due to `error`, but it may be valid in future + */ + case class RecoverableFailedModification(typeId: NetworkObjectTypeId.Value, modifierId: ModifierId, error: Throwable) extends ModificationOutcome - case class SyntacticallyFailedModification(typeId: ModifierTypeId, modifierId: ModifierId, error: Throwable) extends ModificationOutcome + /** + * A signal that block section with id `modifierId` was permanently invalidated during stateless checks + */ + case class SyntacticallyFailedModification(typeId: NetworkObjectTypeId.Value, modifierId: ModifierId, error: Throwable) extends ModificationOutcome /** * Signal associated with stateful validation of a block section */ - case class SemanticallyFailedModification(typeId: ModifierTypeId, modifierId: ModifierId, error: Throwable) extends ModificationOutcome + case class SemanticallyFailedModification(typeId: NetworkObjectTypeId.Value, modifierId: ModifierId, error: Throwable) extends ModificationOutcome /** * Signal associated with stateless validation of a block section */ - case class SyntacticallySuccessfulModifier(typeId: ModifierTypeId, modifierId: ModifierId) extends ModificationOutcome + case class SyntacticallySuccessfulModifier(typeId: NetworkObjectTypeId.Value, modifierId: ModifierId) extends ModificationOutcome /** * Signal sent by node view holder when a full block is applied to state @@ -1312,7 +1316,7 @@ object ErgoNodeViewSynchronizer { */ case class BlockSectionsProcessingCacheUpdate(headersCacheSize: Int, blockSectionsCacheSize: Int, - cleared: (ModifierTypeId, Seq[ModifierId])) + cleared: (NetworkObjectTypeId.Value, Seq[ModifierId])) /** * Command to re-check mempool to clean transactions become invalid while sitting in the mempool up diff --git a/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala b/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala index cef1acd5a8..4d5269819c 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala @@ -5,7 +5,7 @@ import org.ergoplatform.nodeView.history.ErgoHistory.Height import scorex.core.app.Version import scorex.core.consensus.PeerChainStatus import scorex.core.network.ConnectedPeer -import scorex.core.utils.TimeProvider.Time +import org.ergoplatform.nodeView.history.ErgoHistory.Time /** * Container for status of another peer diff --git a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala index 3c0fcec684..02332cb46d 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala @@ -2,18 +2,20 @@ package org.ergoplatform.network import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader, ErgoSyncInfo, ErgoSyncInfoV1, ErgoSyncInfoV2} -import org.ergoplatform.nodeView.history.ErgoHistory.Height -import scorex.core.consensus.{Fork, PeerChainStatus, Older, Unknown} +import org.ergoplatform.nodeView.history.ErgoHistory.{Height, Time} +import scorex.core.consensus.{Fork, Older, PeerChainStatus, Unknown} import scorex.core.network.ConnectedPeer import scorex.core.settings.NetworkSettings -import scorex.core.utils.TimeProvider import scorex.util.ScorexLogging import scala.collection.mutable import scala.concurrent.duration._ import scorex.core.utils.MapPimp -final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: TimeProvider) extends ScorexLogging { +/** + * Data structures and methods to keep status of peers, find ones with expired status to send sync message etc + */ +final case class ErgoSyncTracker(networkSettings: NetworkSettings) extends ScorexLogging { private val MinSyncInterval: FiniteDuration = 20.seconds private val SyncThreshold: FiniteDuration = 1.minute @@ -25,25 +27,34 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: private[network] val statuses = mutable.Map[ConnectedPeer, ErgoPeerStatus]() + /** + * @return get all the current statuses + */ def fullInfo(): Iterable[ErgoPeerStatus] = statuses.values + private def currentTime(): Time = System.currentTimeMillis() + // returns diff def updateLastSyncGetTime(peer: ConnectedPeer): Long = { + val now = currentTime() val prevSyncGetTime = statuses.get(peer).flatMap(_.lastSyncGetTime).getOrElse(0L) - val currentTime = timeProvider.time() + statuses.get(peer).foreach { status => - statuses.update(peer, status.copy(lastSyncGetTime = Option(currentTime))) + statuses.update(peer, status.copy(lastSyncGetTime = Option(now))) } - currentTime - prevSyncGetTime + now - prevSyncGetTime } + /** + * @return true if sync message was sent long time ago to `peer`, or not sent at all yet + */ def notSyncedOrOutdated(peer: ConnectedPeer): Boolean = { val peerOpt = statuses.get(peer) val notSyncedOrMissing = peerOpt.forall(_.lastSyncSentTime.isEmpty) val outdated = peerOpt .flatMap(_.lastSyncSentTime) - .exists(syncTime => (timeProvider.time() - syncTime).millis > SyncThreshold) + .exists(syncTime => (System.currentTimeMillis() - syncTime).millis > SyncThreshold) notSyncedOrMissing || outdated } @@ -107,10 +118,12 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: } } + /** + * Update timestamp of last sync message sent to `peer` + */ def updateLastSyncSentTime(peer: ConnectedPeer): Unit = { - val currentTime = timeProvider.time() statuses.get(peer).foreach { status => - statuses.update(peer, status.copy(lastSyncSentTime = Option(currentTime))) + statuses.update(peer, status.copy(lastSyncSentTime = Option(currentTime()))) } } @@ -118,9 +131,8 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: * Helper method to clear statuses of peers not updated for long enough */ private[network] def clearOldStatuses(): Unit = { - val currentTime = timeProvider.time() val peersToClear = statuses.filter { case (_, status) => - status.lastSyncSentTime.exists(syncTime => (currentTime - syncTime).millis > ClearThreshold) + status.lastSyncSentTime.exists(syncTime => (currentTime() - syncTime).millis > ClearThreshold) }.keys if (peersToClear.nonEmpty) { log.debug(s"Clearing stalled statuses for $peersToClear") @@ -130,9 +142,8 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: } private[network] def outdatedPeers: IndexedSeq[ConnectedPeer] = { - val currentTime = timeProvider.time() statuses.filter { case (_, status) => - status.lastSyncSentTime.exists(syncTime => (currentTime - syncTime).millis > SyncThreshold) + status.lastSyncSentTime.exists(syncTime => (currentTime() - syncTime).millis > SyncThreshold) }.keys.toVector } @@ -168,7 +179,6 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: if (outdated.nonEmpty) { outdated } else { - val currentTime = timeProvider.time() val unknowns = statuses.filter(_._2.status == Unknown).toVector val forks = statuses.filter(_._2.status == Fork).toVector val elders = statuses.filter(_._2.status == Older).toVector @@ -179,8 +189,9 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: unknowns } val nonOutdated = eldersAndUnknown ++ forks + val now = currentTime() nonOutdated.filter { case (_, status) => - (currentTime - status.lastSyncSentTime.getOrElse(0L)).millis >= MinSyncInterval + (now - status.lastSyncSentTime.getOrElse(0L)).millis >= MinSyncInterval }.map(_._1) } diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index 326f63052f..5e13ab0d61 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -4,10 +4,9 @@ import akka.actor.SupervisorStrategy.Escalate import akka.actor.{Actor, ActorRef, ActorSystem, OneForOneStrategy, Props} import org.ergoplatform.ErgoApp import org.ergoplatform.ErgoApp.CriticalSystemException -import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} -import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NetworkObjectTypeId} import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.mempool.ErgoMemPool import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome @@ -22,11 +21,13 @@ import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages._ import org.ergoplatform.modifiers.history.{ADProofs, HistoryModifierSerializer} import scorex.core.consensus.ProgressInfo import scorex.core.settings.ScorexSettings -import scorex.core.utils.{NetworkTimeProvider, ScorexEncoding} +import scorex.core.utils.ScorexEncoding import scorex.core.validation.RecoverableModifierError import scorex.util.{ModifierId, ScorexLogging} import spire.syntax.all.cfor + import java.io.File +import org.ergoplatform.modifiers.history.extension.Extension import scala.annotation.tailrec import scala.collection.mutable @@ -35,12 +36,12 @@ import scala.util.{Failure, Success, Try} /** * Composite local view of the node * - * Contains instances for History, ErgoState, Wallet, MemoryPool. - * Updates them atomically. + * Contains instances for History, ErgoState, Vault, MemoryPool. + * The instances are read-only for external world. + * Updates of the composite view instances are to be performed atomically. * */ -abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSettings, - timeProvider: NetworkTimeProvider) +abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSettings) extends Actor with ScorexLogging with ScorexEncoding with FileUtils { private implicit lazy val actorSystem: ActorSystem = context.system @@ -131,11 +132,11 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti } - protected def requestDownloads(pi: ProgressInfo[BlockSection]): Unit = { + private def requestDownloads(pi: ProgressInfo[BlockSection]): Unit = { //TODO: actually, pi.toDownload contains only 1 modifierid per type, //TODO: see the only case where toDownload is not empty during ProgressInfo construction //TODO: so the code below can be optimized - val toDownload = mutable.Map[ModifierTypeId, Seq[ModifierId]]() + val toDownload = mutable.Map[NetworkObjectTypeId.Value, Seq[ModifierId]]() pi.toDownload.foreach { case (tid, mid) => toDownload.put(tid, toDownload.getOrElse(tid, Seq()) :+ mid) } @@ -143,7 +144,6 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti } - private def trimChainSuffix(suffix: IndexedSeq[BlockSection], rollbackPoint: ModifierId): IndexedSeq[BlockSection] = { val idx = suffix.indexWhere(_.id == rollbackPoint) @@ -235,7 +235,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti updateInfo.state.applyModifier(modToApply, chainTipOpt)(lm => pmodModify(lm.pmod, local = true)) match { case Success(stateAfterApply) => history.reportModifierIsValid(modToApply).map { newHis => - if(modToApply.modifierTypeId == ErgoFullBlock.modifierTypeId) { + if (modToApply.modifierTypeId == ErgoFullBlock.modifierTypeId) { context.system.eventStream.publish(FullBlockApplied(modToApply.asInstanceOf[ErgoFullBlock].header)) } UpdateInformation(newHis, stateAfterApply, None, None, updateInfo.suffix :+ modToApply) @@ -342,7 +342,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti val at0 = System.currentTimeMillis() applyFromCacheLoop(modifiersCache) val at = System.currentTimeMillis() - log.debug(s"Application time: ${at-at0}") + log.debug(s"Application time: ${at - at0}") val cleared = modifiersCache.cleanOverfull() @@ -385,7 +385,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti val state = recreatedState() - val history = ErgoHistory.readOrGenerate(settings, timeProvider) + val history = ErgoHistory.readOrGenerate(settings) val wallet = ErgoWallet.readOrGenerate( history.getReader, settings, LaunchParameters) @@ -403,7 +403,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti def restoreState(): Option[NodeView] = if (ErgoHistory.historyDir(settings).listFiles().isEmpty) { None } else { - val history = ErgoHistory.readOrGenerate(settings, timeProvider) + val history = ErgoHistory.readOrGenerate(settings) log.info("History database read") val memPool = ErgoMemPool.empty(settings) val constants = StateConstants(settings) @@ -424,10 +424,12 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti } //todo: update state in async way? + /** * Remote and local persistent modifiers need to be appended to history, applied to state * which also needs to be git propagated to mempool and wallet - * @param pmod Remote or local persistent modifier + * + * @param pmod Remote or local persistent modifier * @param local whether the modifier was generated locally or not */ protected def pmodModify(pmod: BlockSection, local: Boolean): Unit = { @@ -462,7 +464,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti val fullBlockHeight = newHistory.fullBlockHeight val almostSynced = (headersHeight - fullBlockHeight) < almostSyncedGap - val newMemPool = if(almostSynced) { + val newMemPool = if (almostSynced) { updateMemPool(progressInfo.toRemove, blocksApplied, memoryPool()) } else { memoryPool() @@ -496,7 +498,11 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti log.info(s"Persistent modifier ${pmod.encodedId} applied successfully") updateNodeView(Some(newHistory), Some(newMinState), Some(newVault), Some(newMemPool)) chainProgress = - Some(ChainProgress(pmod, headersHeight, fullBlockHeight, timeProvider.time())) + Some(ChainProgress(pmod, headersHeight, fullBlockHeight, System.currentTimeMillis())) + + if (progressInfo.chainSwitchingNeeded) { + context.system.eventStream.publish(Rollback(progressInfo.branchPoint.get)) + } case Failure(e) => log.warn(s"Can`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod) to minimal state", e) updateNodeView(updatedHistory = Some(newHistory)) @@ -531,7 +537,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti ErgoState.readOrGenerate(settings, constants) .asInstanceOf[State] .ensuring( - state => java.util.Arrays.equals(state.rootHash, settings.chainSettings.genesisStateDigest), + state => java.util.Arrays.equals(state.rootDigest, settings.chainSettings.genesisStateDigest), "State root is incorrect" ) } @@ -652,11 +658,11 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti if (mempool) sender() ! ChangedMempool(nodeView._4.getReader) } - protected def handleHealthCheck: Receive = { + private def handleHealthCheck: Receive = { case IsChainHealthy => log.info(s"Check that chain is healthy, progress is $chainProgress") val healthCheckReply = chainProgress.map { progress => - ErgoNodeViewHolder.checkChainIsHealthy(progress, history(), timeProvider, settings) + ErgoNodeViewHolder.checkChainIsHealthy(progress, history(), settings) }.getOrElse(ChainIsStuck("Node already stuck when started")) sender() ! healthCheckReply } @@ -717,22 +723,26 @@ object ErgoNodeViewHolder { case class BlockAppliedTransactions(txs: Seq[ModifierId]) extends NodeViewHolderEvent - case class DownloadRequest(modifiersToFetch: Map[ModifierTypeId, Seq[ModifierId]]) extends NodeViewHolderEvent + /** + * When node view holder is realizing it knows IDs of block sections not downloaded yte, it sends this signal + * to download them + */ + case class DownloadRequest(modifiersToFetch: Map[NetworkObjectTypeId.Value, Seq[ModifierId]]) extends NodeViewHolderEvent case class CurrentView[State](history: ErgoHistory, state: State, vault: ErgoWallet, pool: ErgoMemPool) /** * Checks whether chain got stuck by comparing timestamp of bestFullBlock or last time a modifier was applied to history. + * * @param progress metadata of last chain update * @return ChainIsHealthy if chain is healthy and ChainIsStuck(error) with details if it got stuck */ def checkChainIsHealthy( progress: ChainProgress, history: ErgoHistory, - timeProvider: NetworkTimeProvider, settings: ErgoSettings): HealthCheckResult = { val ChainProgress(lastMod, headersHeight, blockHeight, lastUpdate) = progress - val chainUpdateDelay = timeProvider.time() - lastUpdate + val chainUpdateDelay = System.currentTimeMillis() - lastUpdate val acceptableChainUpdateDelay = settings.nodeSettings.acceptableChainUpdateDelay def chainUpdateDelayed = chainUpdateDelay > acceptableChainUpdateDelay.toMillis def chainSynced = @@ -754,34 +764,29 @@ object ErgoNodeViewHolder { } } -private[nodeView] class DigestNodeViewHolder(settings: ErgoSettings, - timeProvider: NetworkTimeProvider) - extends ErgoNodeViewHolder[DigestState](settings, timeProvider) +private[nodeView] class DigestNodeViewHolder(settings: ErgoSettings) + extends ErgoNodeViewHolder[DigestState](settings) -private[nodeView] class UtxoNodeViewHolder(settings: ErgoSettings, - timeProvider: NetworkTimeProvider) - extends ErgoNodeViewHolder[UtxoState](settings, timeProvider) +private[nodeView] class UtxoNodeViewHolder(settings: ErgoSettings) + extends ErgoNodeViewHolder[UtxoState](settings) object ErgoNodeViewRef { - private def digestProps(settings: ErgoSettings, - timeProvider: NetworkTimeProvider): Props = - Props.create(classOf[DigestNodeViewHolder], settings, timeProvider) + private def digestProps(settings: ErgoSettings): Props = + Props.create(classOf[DigestNodeViewHolder], settings) - private def utxoProps(settings: ErgoSettings, - timeProvider: NetworkTimeProvider): Props = - Props.create(classOf[UtxoNodeViewHolder], settings, timeProvider) + private def utxoProps(settings: ErgoSettings): Props = + Props.create(classOf[UtxoNodeViewHolder], settings) - def props(settings: ErgoSettings, - timeProvider: NetworkTimeProvider): Props = + private def props(settings: ErgoSettings): Props = (settings.nodeSettings.stateType match { - case StateType.Digest => digestProps(settings, timeProvider) - case StateType.Utxo => utxoProps(settings, timeProvider) + case StateType.Digest => digestProps(settings) + case StateType.Utxo => utxoProps(settings) }).withDispatcher("critical-dispatcher") - def apply(settings: ErgoSettings, timeProvider: NetworkTimeProvider)(implicit system: ActorSystem): ActorRef = - system.actorOf(props(settings, timeProvider)) + def apply(settings: ErgoSettings)(implicit system: ActorSystem): ActorRef = + system.actorOf(props(settings)) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala index e9f3f151aa..c83d82e63e 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala @@ -1,18 +1,19 @@ package org.ergoplatform.nodeView.history -import java.io.File +import akka.actor.ActorContext +import java.io.File import org.ergoplatform.ErgoLikeContext import org.ergoplatform.mining.AutolykosPowScheme import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.header.{Header, PreGenesisHeader} -import org.ergoplatform.modifiers.{NonHeaderBlockSection, ErgoFullBlock, BlockSection} +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NonHeaderBlockSection} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.ReceivableMessages.StartExtraIndexer import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.nodeView.history.storage.modifierprocessors._ import org.ergoplatform.settings._ import org.ergoplatform.utils.LoggingUtil import scorex.core.consensus.ProgressInfo -import scorex.core.utils.NetworkTimeProvider import scorex.core.validation.RecoverableModifierError import scorex.util.{ModifierId, ScorexLogging, idToBytes} @@ -95,19 +96,19 @@ trait ErgoHistory modifier match { case fb: ErgoFullBlock => val nonMarkedIds = (fb.header.id +: fb.header.sectionIds.map(_._2)) - .filter(id => historyStorage.getIndex(validityKey(id)).isEmpty) + .filter(id => historyStorage.getIndex(validityKey(id)).isEmpty).toArray if (nonMarkedIds.nonEmpty) { historyStorage.insert( nonMarkedIds.map(id => validityKey(id) -> Array(1.toByte)), - Nil).map(_ => this) + Array.empty[BlockSection]).map(_ => this) } else { Success(this) } case _ => historyStorage.insert( Array(validityKey(modifier.id) -> Array(1.toByte)), - Nil).map(_ => this) + Array.empty[BlockSection]).map(_ => this) } } @@ -124,17 +125,17 @@ trait ErgoHistory log.warn(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as invalid") correspondingHeader(modifier) match { case Some(invalidatedHeader) => - val invalidatedHeaders = continuationHeaderChains(invalidatedHeader, _ => true).flatten.distinct + val invalidatedHeaders = continuationHeaderChains(invalidatedHeader, _ => true).flatten.distinct.toArray val invalidatedIds = invalidatedHeaders.map(_.id).toSet val validityRow = invalidatedHeaders.flatMap(h => Seq(h.id, h.transactionsId, h.ADProofsId) .map(id => validityKey(id) -> Array(0.toByte))) - log.info(s"Going to invalidate ${invalidatedHeader.encodedId} and ${invalidatedHeaders.map(_.encodedId)}") + log.info(s"Going to invalidate ${invalidatedHeader.encodedId} and ${invalidatedHeaders.map(_.encodedId).mkString("Array(", ", ", ")")}") val bestHeaderIsInvalidated = bestHeaderIdOpt.exists(id => invalidatedIds.contains(id)) val bestFullIsInvalidated = bestFullBlockIdOpt.exists(id => invalidatedIds.contains(id)) (bestHeaderIsInvalidated, bestFullIsInvalidated) match { case (false, false) => // Modifiers from best header and best full chain are not involved, no rollback and links change required - historyStorage.insert(validityRow, Nil).map { _ => + historyStorage.insert(validityRow, Array.empty[BlockSection]).map { _ => this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) } case _ => @@ -144,8 +145,8 @@ trait ErgoHistory if (!bestFullIsInvalidated) { //Only headers chain involved historyStorage.insert( - newBestHeaderOpt.map(h => BestHeaderKey -> idToBytes(h.id)).toSeq, - Seq.empty + newBestHeaderOpt.map(h => BestHeaderKey -> idToBytes(h.id)).toArray, + Array.empty[BlockSection] ).map { _ => this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) } @@ -173,7 +174,7 @@ trait ErgoHistory val changedLinks = validHeadersChain.lastOption.map(b => BestFullBlockKey -> idToBytes(b.id)) ++ newBestHeaderOpt.map(h => BestHeaderKey -> idToBytes(h.id)).toSeq val toInsert = validityRow ++ changedLinks ++ chainStatusRow - historyStorage.insert(toInsert, Seq.empty).map { _ => + historyStorage.insert(toInsert, Array.empty[BlockSection]).map { _ => val toRemove = if (genesisInvalidated) invalidatedChain else invalidatedChain.tail this -> ProgressInfo(Some(branchPointHeader.id), toRemove, validChain, Seq.empty) } @@ -182,7 +183,7 @@ trait ErgoHistory case None => //No headers become invalid. Just mark this modifier as invalid log.warn(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is missing corresponding header") - historyStorage.insert(Array(validityKey(modifier.id) -> Array(0.toByte)), Nil).map { _ => + historyStorage.insert(Array(validityKey(modifier.id) -> Array(0.toByte)), Array.empty[BlockSection]).map { _ => this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) } } @@ -232,6 +233,13 @@ trait ErgoHistory object ErgoHistory extends ScorexLogging { + /** + * Type for time, represents machine-specific timestamp of a transaction + * or block section, as miliseconds passed since beginning of UNIX + * epoch on the machine + */ + type Time = Long + type Height = ErgoLikeContext.Height // Int type Score = BigInt type Difficulty = BigInt @@ -262,14 +270,17 @@ object ErgoHistory extends ScorexLogging { afterHeaders.map { hId => history.forgetHeader(hId) } - history.historyStorage.remove(Array(history.heightIdsKey(bestHeaderHeight + 1)), Nil) + history.historyStorage.remove(Array(history.heightIdsKey(bestHeaderHeight + 1)), Array.empty[ModifierId]) true } else { false } } - def readOrGenerate(ergoSettings: ErgoSettings, ntp: NetworkTimeProvider): ErgoHistory = { + /** + * @return ErgoHistory instance with new database or database read from existing folder + */ + def readOrGenerate(ergoSettings: ErgoSettings)(implicit context: ActorContext): ErgoHistory = { val db = HistoryStorage(ergoSettings) val nodeSettings = ergoSettings.nodeSettings @@ -279,7 +290,6 @@ object ErgoHistory extends ScorexLogging { override protected val settings: ErgoSettings = ergoSettings override protected[history] val historyStorage: HistoryStorage = db override val powScheme: AutolykosPowScheme = chainSettings.powScheme - override protected val timeProvider: NetworkTimeProvider = ntp } case (false, true) => @@ -287,7 +297,6 @@ object ErgoHistory extends ScorexLogging { override protected val settings: ErgoSettings = ergoSettings override protected[history] val historyStorage: HistoryStorage = db override val powScheme: AutolykosPowScheme = chainSettings.powScheme - override protected val timeProvider: NetworkTimeProvider = ntp } case (true, false) => @@ -295,7 +304,6 @@ object ErgoHistory extends ScorexLogging { override protected val settings: ErgoSettings = ergoSettings override protected[history] val historyStorage: HistoryStorage = db override val powScheme: AutolykosPowScheme = chainSettings.powScheme - override protected val timeProvider: NetworkTimeProvider = ntp } case (false, false) => @@ -303,13 +311,14 @@ object ErgoHistory extends ScorexLogging { override protected val settings: ErgoSettings = ergoSettings override protected[history] val historyStorage: HistoryStorage = db override val powScheme: AutolykosPowScheme = chainSettings.powScheme - override protected val timeProvider: NetworkTimeProvider = ntp } } repairIfNeeded(history) log.info("History database read") + if(ergoSettings.nodeSettings.extraIndex) // start extra indexer, if enabled + context.system.eventStream.publish(StartExtraIndexer(history)) history } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala index ca5352f300..47b5dc6c43 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala @@ -4,12 +4,13 @@ import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.{Header, PreGenesisHeader} import org.ergoplatform.modifiers.history.popow.{NipopowAlgos, NipopowProof, PoPowHeader, PoPowParams} -import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NonHeaderBlockSection} +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NetworkObjectTypeId, NonHeaderBlockSection} import org.ergoplatform.nodeView.history.ErgoHistory.Height +import org.ergoplatform.nodeView.history.extra.ExtraIndex import org.ergoplatform.nodeView.history.storage._ import org.ergoplatform.nodeView.history.storage.modifierprocessors._ import org.ergoplatform.settings.ErgoSettings -import scorex.core.{ModifierTypeId, NodeViewComponent} +import scorex.core.NodeViewComponent import scorex.core.consensus.{ContainsModifiers, Equal, Fork, ModifierSemanticValidity, Older, PeerChainStatus, Unknown, Younger} import scorex.core.utils.ScorexEncoding import scorex.core.validation.MalformedModifierError @@ -30,7 +31,7 @@ trait ErgoHistoryReader with ScorexLogging with ScorexEncoding { - type ModifierIds = Seq[(ModifierTypeId, ModifierId)] + type ModifierIds = Seq[(NetworkObjectTypeId.Value, ModifierId)] protected[history] val historyStorage: HistoryStorage @@ -73,7 +74,7 @@ trait ErgoHistoryReader * @param id - modifier id * @return type and raw bytes of semantically valid ErgoPersistentModifier with the given id it is in history */ - def modifierTypeAndBytesById(id: ModifierId): Option[(ModifierTypeId, Array[Byte])] = + def modifierTypeAndBytesById(id: ModifierId): Option[(NetworkObjectTypeId.Value, Array[Byte])] = if (isSemanticallyValid(id) != ModifierSemanticValidity.Invalid) { historyStorage.modifierTypeAndBytesById(id) } else { @@ -95,13 +96,24 @@ trait ErgoHistoryReader * * @param id - modifier id * @tparam T - expected Type - * @return semantically valid ErgoPersistentModifier of type T with the given id it is in history + * @return semantically valid ErgoPersistentModifier of type T with the given id if it is in history */ def typedModifierById[T <: BlockSection : ClassTag](id: ModifierId): Option[T] = modifierById(id) match { case Some(m: T) => Some(m) case _ => None } + /** Get index of expected type by its identifier + * @param id - index id + * @tparam T - expected Type + * @return index of type T with the given id if it is in history + */ + def typedExtraIndexById[T <: ExtraIndex : ClassTag](id: ModifierId): Option[T] = + historyStorage.getExtraIndex(id) match { + case Some(m: T) => Some(m) + case _ => None + } + override def contains(id: ModifierId): Boolean = historyStorage.contains(id) /** diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala new file mode 100644 index 0000000000..10c0cfd364 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -0,0 +1,85 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} +import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.fastIdToBytes +import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.uniqueId +import scorex.core.serialization.ScorexSerializer +import scorex.util.{ModifierId, ScorexLogging, bytesToId} +import scorex.util.serialization.{Reader, Writer} +import spire.implicits.cfor + +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer + +class BalanceInfo(var nanoErgs: Long = 0L, + val tokens: ArrayBuffer[(ModifierId,Long)] = ArrayBuffer.empty[(ModifierId,Long)]) extends ScorexLogging { + + val additionalTokenInfo: mutable.HashMap[ModifierId,(String,Int)] = mutable.HashMap.empty[ModifierId,(String,Int)] + + def retreiveAdditionalTokenInfo(history: ErgoHistoryReader): BalanceInfo = { + additionalTokenInfo ++= tokens.map(token => { + val iT: IndexedToken = history.typedExtraIndexById[IndexedToken](uniqueId(token._1)).get + (token._1,(iT.name,iT.decimals)) + }) + this + } + + private def index(id: ModifierId): Option[Int] = { + cfor(0)(_ < tokens.length, _ + 1) { i => + if(tokens(i)._1 == id) return Some(i) + } + None + } + + def add(box: ErgoBox): Unit = { + nanoErgs += box.value + cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => + val id: ModifierId = bytesToId(box.additionalTokens(i)._1) + index(id) match { + case Some(n) => tokens(n) = Tuple2(id, tokens(n)._2 + box.additionalTokens(i)._2) + case None => tokens += Tuple2(id, box.additionalTokens(i)._2) + } + } + } + + def subtract(box: ErgoBox)(implicit ae: ErgoAddressEncoder): Unit = { + nanoErgs = math.max(nanoErgs - box.value, 0) + cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => + val id: ModifierId = bytesToId(box.additionalTokens(i)._1) + index(id) match { + case Some(n) => + val newVal: Long = tokens(n)._2 - box.additionalTokens(i)._2 + if(newVal == 0) + tokens.remove(n) + else + tokens(n) = (id, newVal) + case None => log.warn(s"Failed to subtract token $id from address ${ae.fromProposition(box.ergoTree).map(ae.toString).getOrElse(box.ergoTree.bytesHex)}") + } + + } + } + +} + +object BalanceInfoSerializer extends ScorexSerializer[BalanceInfo] { + + override def serialize(bI: BalanceInfo, w: Writer): Unit = { + w.putLong(bI.nanoErgs) + w.putInt(bI.tokens.length) + cfor(0)(_ < bI.tokens.length, _ + 1) { i => + w.putBytes(fastIdToBytes(bI.tokens(i)._1)) + w.putLong(bI.tokens(i)._2) + } + } + + override def parse(r: Reader): BalanceInfo = { + val bI: BalanceInfo = new BalanceInfo(r.getLong()) + val tokensLen: Int = r.getInt() + cfor(0)(_ < tokensLen, _ + 1) { _ => + bI.tokens += Tuple2(bytesToId(r.getBytes(32)), r.getLong()) + } + bI + } + +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala new file mode 100644 index 0000000000..0c22fe4956 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala @@ -0,0 +1,51 @@ +package org.ergoplatform.nodeView.history.extra + +import scorex.core.serialization.ScorexSerializer +import scorex.util.serialization.{Reader, Writer} + +object ExtraIndexSerializer extends ScorexSerializer[ExtraIndex]{ + + override def serialize(obj: ExtraIndex, w: Writer): Unit = { + obj match { + case m: IndexedErgoAddress => + w.put(IndexedErgoAddress.extraIndexTypeId) + IndexedErgoAddressSerializer.serialize(m, w) + case m: IndexedErgoTransaction => + w.put(IndexedErgoTransaction.extraIndexTypeId) + IndexedErgoTransactionSerializer.serialize(m, w) + case m: IndexedErgoBox => + w.put(IndexedErgoBox.extraIndexTypeId) + IndexedErgoBoxSerializer.serialize(m, w) + case m: NumericTxIndex => + w.put(NumericTxIndex.extraIndexTypeId) + NumericTxIndexSerializer.serialize(m, w) + case m: NumericBoxIndex => + w.put(NumericBoxIndex.extraIndexTypeId) + NumericBoxIndexSerializer.serialize(m, w) + case m: IndexedToken => + w.put(IndexedToken.extraIndexTypeId) + IndexedTokenSerializer.serialize(m, w) + case m => + throw new Error(s"Serialization for unknown index: $m") + } + } + + override def parse(r: Reader): ExtraIndex = { + r.getByte() match { + case IndexedErgoAddress.`extraIndexTypeId` => + IndexedErgoAddressSerializer.parse(r) + case IndexedErgoTransaction.`extraIndexTypeId` => + IndexedErgoTransactionSerializer.parse(r) + case IndexedErgoBox.`extraIndexTypeId` => + IndexedErgoBoxSerializer.parse(r) + case NumericTxIndex.`extraIndexTypeId` => + NumericTxIndexSerializer.parse(r) + case NumericBoxIndex.`extraIndexTypeId` => + NumericBoxIndexSerializer.parse(r) + case IndexedToken.`extraIndexTypeId` => + IndexedTokenSerializer.parse(r) + case m => + throw new Error(s"Deserialization for unknown type byte: $m") + } + } + } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala new file mode 100644 index 0000000000..f595c30651 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -0,0 +1,462 @@ +package org.ergoplatform.nodeView.history.extra + +import akka.actor.{Actor, ActorRef, ActorSystem, Props} +import org.ergoplatform.ErgoBox.{BoxId, TokenId} +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, ErgoBox, Pay2SAddress} +import org.ergoplatform.modifiers.history.BlockTransactions +import org.ergoplatform.modifiers.history.header.Header +import org.ergoplatform.modifiers.mempool.ErgoTransaction +import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.{FullBlockApplied, Rollback} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{GlobalBoxIndexKey, GlobalTxIndexKey, IndexedHeightKey, getIndex} +import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.ReceivableMessages.StartExtraIndexer +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.segmentTreshold +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.hashErgoTree +import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.tokenRegistersSet +import org.ergoplatform.nodeView.history.storage.HistoryStorage +import org.ergoplatform.settings.{Algos, CacheSettings, ChainSettings} +import scorex.util.{ModifierId, ScorexLogging, bytesToId} +import sigmastate.Values.ErgoTree + +import java.nio.ByteBuffer +import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import spire.syntax.all.cfor + +/** + * Base trait for extra indexer actor and its test. + */ +trait ExtraIndexerBase extends ScorexLogging { + + // Indexed block height + protected var indexedHeight: Int = 0 + private val indexedHeightBuffer: ByteBuffer = ByteBuffer.allocate(4) + + // Indexed transaction count + protected var globalTxIndex: Long = 0L + private val globalTxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) + + // Indexed box count + protected var globalBoxIndex: Long = 0L + private val globalBoxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) + + // Last block height when buffer contents were saved to database + protected var lastWroteToDB: Int = 0 + + // Max buffer size (determined by config) + protected val saveLimit: Int + + // Address encoder instance + protected implicit val addressEncoder: ErgoAddressEncoder + + // Flag to signal when indexer has reached current block height + protected var caughtUp: Boolean = false + + // Flag to signal a rollback + protected var rollback: Boolean = false + + // Database handle + protected var _history: ErgoHistory = null + + protected def chainHeight: Int = _history.fullBlockHeight + protected def history: ErgoHistoryReader = _history.getReader + protected def historyStorage: HistoryStorage = _history.historyStorage + + // fast access buffers + private val general: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex] + private val boxes: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] + private val trees: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] + + // input tokens in a tx + protected val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] + + /** + * Find a box in the boxes buffer. + * + * @param id - id of the wanted box + * @return an Option containing the index of the wanted box in the boxes buffer or None if box was not found + */ + private def findBox(id: BoxId): Option[Int] = { + cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first + if (java.util.Arrays.equals(boxes(i).serializedId, id)) + return Some(i) // box found in last saveLimit modifiers + } + None + } + + /** + * Spend an IndexedErgoBox from buffer or database. Also record tokens for later use in balance tracking logic. + * + * @param id - id of the wanted box + * @param txId - id of the spending transaction + * @param height - height of the block the spending transaction is included in + * @return index of spent box in boxes buffer or -1 if an unknown box was spent + */ + private def findAndSpendBox(id: BoxId, txId: ModifierId, height: Int): Int = { + findBox(id) match { + case Some(i) => + tokens ++= boxes(i).asSpent(txId, height).box.additionalTokens.toArray + i + case None => + history.typedExtraIndexById[IndexedErgoBox](bytesToId(id)) match { // box not found in last saveLimit modifiers + case Some(x) => // box found in DB, update + boxes += x.asSpent(txId, height) + tokens ++= x.box.additionalTokens.toArray + boxes.length - 1 + case None => // box not found at all (this shouldn't happen) + log.warn(s"Unknown box used as input: ${bytesToId(id)}") + -1 + } + } + } + + /** + * Add or subtract a box from an address in the buffer or in database. + * + * @param id - hash of the (ergotree) address + * @param spendOrReceive - ErgoBox to spend or IndexedErgoBox to receive + * @return index of updated tree in buffer or -1 if the tree was unknown + */ + private def findAndUpdateTree(id: ModifierId, spendOrReceive: Either[ErgoBox, IndexedErgoBox]): Int = { + cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first + if (trees(i).id == id) { // address found in last saveLimit modifiers + spendOrReceive match { + case Left(box) => trees(i).addTx(globalTxIndex).spendBox(box) // spend box + case Right(iEb) => trees(i).addTx(globalTxIndex).addBox(iEb) // receive box + } + return i + } + } + history.typedExtraIndexById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers + case Some(x) => + spendOrReceive match { + case Left(box) => trees += x.addTx(globalTxIndex).spendBox(box) // spend box + case Right(iEb) => trees += x.addTx(globalTxIndex).addBox(iEb) // receive box + } + trees.length - 1 + case None => // address not found at all + spendOrReceive match { + case Left(box) => log.warn(s"Unknown address spent box ${bytesToId(box.id)}") // spend box should never happen by an unknown address + case Right(iEb) => trees += IndexedErgoAddress(id, ListBuffer(globalTxIndex), ListBuffer.empty[Long], Some(new BalanceInfo)).addBox(iEb) // receive box + } + -1 + } + } + + /** + * @return number of indexes in all buffers + */ + private def modCount: Int = general.length + boxes.length + trees.length + + /** + * Write buffered indexes to database and clear buffers. + */ + private def saveProgress(writeLog: Boolean = true): Unit = { + + if(modCount == 0) return + + val start: Long = System.nanoTime() + + // perform segmentation on big modifiers + val addressesLen: Int = trees.length + cfor(0)(_ < addressesLen, _ + 1) { i => + if (trees(i).txs.length > segmentTreshold || trees(i).boxes.length > segmentTreshold) trees ++= trees(i).splitToSegments() + } + + // merge all modifiers to an Array, avoids reallocations during concatenation (++) + val all: Array[ExtraIndex] = new Array[ExtraIndex](modCount) + val offset: Array[Int] = Array(0, general.length, general.length + boxes.length) + cfor(0)(_ < general.length, _ + 1) { i => all(i + offset(0)) = general(i) } + cfor(0)(_ < boxes.length, _ + 1) { i => all(i + offset(1)) = boxes(i) } + cfor(0)(_ < trees.length, _ + 1) { i => all(i + offset(2)) = trees(i) } + + // insert modifiers and progress info to db + indexedHeightBuffer.clear() + globalTxIndexBuffer.clear() + globalBoxIndexBuffer.clear() + historyStorage.insertExtra(Array((IndexedHeightKey, indexedHeightBuffer.putInt(indexedHeight).array), + (GlobalTxIndexKey, globalTxIndexBuffer.putLong(globalTxIndex).array), + (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), all) + + val end: Long = System.nanoTime() + + if (writeLog) + log.info(s"Processed ${trees.length} ErgoTrees with ${boxes.length} boxes and inserted them to database in ${(end - start) / 1000000D}ms") + + // clear buffers for next batch + general.clear() + boxes.clear() + trees.clear() + + lastWroteToDB = indexedHeight + } + + /** + * Process a batch of BlockTransactions into memory and occasionally write them to database. + * + * @param bt - BlockTransaction to process + * @param height - height of the block containing the transactions + */ + protected def index(bt: BlockTransactions, height: Int): Unit = { + + if (rollback || // rollback in progress + (caughtUp && height <= indexedHeight)) // do not process older blocks again after caught up (due to actor message queue) + return + + var boxCount: Int = 0 + + // record transactions and boxes + cfor(0)(_ < bt.txs.length, _ + 1) { n => + + val tx: ErgoTransaction = bt.txs(n) + val inputs: Array[Long] = Array.ofDim[Long](tx.inputs.length) + + tokens.clear() + + //process transaction inputs + if (height != 1) { //only after 1st block (skip genesis box) + cfor(0)(_ < tx.inputs.size, _ + 1) { i => + val boxIndex: Int = findAndSpendBox(tx.inputs(i).boxId, tx.id, height) + if (boxIndex >= 0) { // spend box and add tx + findAndUpdateTree(bytesToId(hashErgoTree(boxes(boxIndex).box.ergoTree)), Left(boxes(boxIndex).box)) + inputs(i) = boxes(boxIndex).globalIndex + } + } + } + + //process transaction outputs + cfor(0)(_ < tx.outputs.size, _ + 1) { i => + val box: ErgoBox = tx.outputs(i) + boxes += new IndexedErgoBox(height, None, None, box, globalBoxIndex) // box by id + general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number + + // box by address + findAndUpdateTree(bytesToId(hashErgoTree(box.ergoTree)), Right(boxes(findBox(box.id).get))) + + // check if box is creating a new token, if yes record it + if (tokenRegistersSet(box)) + cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => + if (!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) { + general += IndexedTokenSerializer.fromBox(box) + } + } + + globalBoxIndex += 1 + boxCount += 1 + + } + + //process transaction + general += IndexedErgoTransaction(tx.id, height, globalTxIndex, inputs) + general += NumericTxIndex(globalTxIndex, tx.id) + + globalTxIndex += 1 + + } + + log.info(s"Buffered block $height / $chainHeight [txs: ${bt.txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") + + if (caughtUp) { + + indexedHeight = height // update height here after caught up with chain + + if (modCount >= saveLimit || // modifier limit reached to write to db + history.fullBlockHeight == history.headersHeight) // write to db every block after chain synced + saveProgress() + + } else if (modCount >= saveLimit) + saveProgress() // active syncing, write to db after modifier limit + + } + + /** + * Main indexer loop that tries to catch up with the already present blocks in database. + */ + protected def run(): Unit = { + + indexedHeight = getIndex(IndexedHeightKey)(history).getInt + globalTxIndex = getIndex(GlobalTxIndexKey)(history).getLong + globalBoxIndex = getIndex(GlobalBoxIndexKey)(history).getLong + + log.info(s"Started extra indexer at height $indexedHeight") + + while (indexedHeight < chainHeight && !rollback) { + indexedHeight += 1 + index(history.bestBlockTransactionsAt(indexedHeight).get, indexedHeight) + } + + saveProgress(false) // flush any remaining data + + if (rollback) + log.info("Stopping indexer to perform rollback") + else { + caughtUp = true + log.info("Indexer caught up with chain") + } + + } + + /** + * Remove all indexes after a given height and revert address balances. + * + * @param height - starting height + */ + protected def removeAfter(height: Int): Unit = { + + saveProgress(false) + log.info(s"Rolling back indexes from $indexedHeight to $height") + + val lastTxToKeep: ErgoTransaction = history.bestBlockTransactionsAt(height).get.txs.last + + // remove all tx indexes + val txTarget: Long = history.typedExtraIndexById[IndexedErgoTransaction](lastTxToKeep.id).get.globalIndex + val txs: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId] + globalTxIndex -= 1 + while(globalTxIndex > txTarget) { + val tx: IndexedErgoTransaction = NumericTxIndex.getTxByNumber(history, globalTxIndex).get + tx.inputNums.map(NumericBoxIndex.getBoxByNumber(history, _).get).foreach(iEb => { // undo all spendings + iEb.spendingHeightOpt = None + iEb.spendingTxIdOpt = None + val address: IndexedErgoAddress = history.typedExtraIndexById[IndexedErgoAddress](bytesToId(hashErgoTree(iEb.box.ergoTree))).get.addBox(iEb, false) + historyStorage.insertExtra(Array.empty, Array(iEb, address)) + }) + txs += tx.id // tx by id + txs += bytesToId(NumericTxIndex.indexToBytes(globalTxIndex)) // tx id by number + globalTxIndex -= 1 + } + globalTxIndex += 1 + historyStorage.removeExtra(txs.toArray) + + // remove all box indexes, tokens and address balances + val boxTarget: Long = history.typedExtraIndexById[IndexedErgoBox](bytesToId(lastTxToKeep.outputs.last.id)).get.globalIndex + val toRemove: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId] + globalBoxIndex -= 1 + while(globalBoxIndex > boxTarget) { + val iEb: IndexedErgoBox = NumericBoxIndex.getBoxByNumber(history, globalBoxIndex).get + val address: IndexedErgoAddress = history.typedExtraIndexById[IndexedErgoAddress](bytesToId(hashErgoTree(iEb.box.ergoTree))).get + address.spendBox(iEb.box) + if(tokenRegistersSet(iEb.box)) + history.typedExtraIndexById[IndexedToken](IndexedTokenSerializer.fromBox(iEb.box).id) match { + case Some(token) => toRemove += token.id // token created, delete + case None => // no token created + } + address.rollback(txTarget, boxTarget)(_history) + toRemove += iEb.id // box by id + toRemove += bytesToId(NumericBoxIndex.indexToBytes(globalBoxIndex)) // box id by number + globalBoxIndex -= 1 + } + globalBoxIndex += 1 + historyStorage.removeExtra(toRemove.toArray) + + saveProgress(false) + + log.info(s"Successfully rolled back indexes to $height") + + rollback = false + + } + +} + + +/** + * Actor that constructs an index of database elements. + * @param cacheSettings - cacheSettings to use for saveLimit size + */ +class ExtraIndexer(cacheSettings: CacheSettings, + ae: ErgoAddressEncoder) + extends Actor with ExtraIndexerBase { + + override val saveLimit: Int = cacheSettings.history.extraCacheSize * 10 + + override implicit val addressEncoder: ErgoAddressEncoder = ae + + override def preStart(): Unit = { + context.system.eventStream.subscribe(self, classOf[FullBlockApplied]) + context.system.eventStream.subscribe(self, classOf[Rollback]) + context.system.eventStream.subscribe(self, classOf[StartExtraIndexer]) + } + + override def postStop(): Unit = + log.info(s"Stopped extra indexer at height ${if(lastWroteToDB > 0) lastWroteToDB else indexedHeight}") + + override def receive: Receive = { + + case FullBlockApplied(header: Header) if caughtUp => + index(history.typedModifierById[BlockTransactions](header.transactionsId).get, header.height) // after the indexer caught up with the chain, stay up to date + + case Rollback(branchPoint: ModifierId) => + val branchHeight: Int = history.heightOf(branchPoint).get + rollback = branchHeight < indexedHeight + if(rollback) { + removeAfter(branchHeight) + run() // restart indexer + } + + case StartExtraIndexer(history: ErgoHistory) => + _history = history + run() + + } +} + +object ExtraIndexer { + + type ExtraIndexTypeId = Byte + + object ReceivableMessages { + /** + * Initialize ExtraIndexer and start indexing. + * @param history - handle to database + */ + case class StartExtraIndexer(history: ErgoHistory) + } + + /** + * @return address constructed from the ErgoTree of this box + */ + def getAddress(tree: ErgoTree)(implicit ae: ErgoAddressEncoder): ErgoAddress = + tree.root match { + case Right(_) => ae.fromProposition(tree).get // default most of the time + case Left(_) => new Pay2SAddress(tree, tree.bytes) // needed for burn address 4MQyMKvMbnCJG3aJ + } + + private val hexIndex: Array[Byte] = { + val index = Array.fill[Byte](128)(0xff.toByte) + "0123456789abcdef".toCharArray.zipWithIndex.foreach { case (c, i) => + index(c) = i.toByte + } + "abcdef".toCharArray.foreach{ c => + index(c.toUpper) = index(c) + } + index + } + + /** + * Faster id to bytes - no safety checks + * @param id - ModifierId to convert to byte representation + * @return an array of bytes + */ + private[extra] def fastIdToBytes(id: ModifierId): Array[Byte] = { + val x: Array[Byte] = new Array[Byte](id.length / 2) + cfor(0)(_ < id.length, _ + 2) {i => x(i / 2) = ((hexIndex(id(i)) << 4) | hexIndex(id(i + 1))).toByte} + x + } + + val IndexedHeightKey: Array[Byte] = Algos.hash("indexed height") + val GlobalTxIndexKey: Array[Byte] = Algos.hash("txns height") + val GlobalBoxIndexKey: Array[Byte] = Algos.hash("boxes height") + + def getIndex(key: Array[Byte])(history: ErgoHistoryReader): ByteBuffer = + ByteBuffer.wrap(history.modifierBytesById(bytesToId(key)).getOrElse(Array.fill[Byte](8){0})) + + def apply(chainSettings: ChainSettings, cacheSettings: CacheSettings)(implicit system: ActorSystem): ActorRef = + system.actorOf(Props.create(classOf[ExtraIndexer], cacheSettings, chainSettings.addressEncoder)) +} + +/** + * Base trait for all additional indexes made by ExtraIndexer + */ +trait ExtraIndex { + def id: ModifierId = bytesToId(serializedId) + def serializedId: Array[Byte] +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala new file mode 100644 index 0000000000..4a9f0f1103 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -0,0 +1,299 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} +import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes} +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getBoxes, getSegmentsForRange, getTxs, segmentTreshold, slice} +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, txSegmentId} +import org.ergoplatform.settings.Algos +import scorex.core.serialization.ScorexSerializer +import scorex.util.{ModifierId, ScorexLogging, bytesToId} +import scorex.util.serialization.{Reader, Writer} +import sigmastate.Values.ErgoTree + +import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import spire.syntax.all.cfor + +/** + * An index of an address (ErgoTree) + * @param treeHash - hash of the corresponding ErgoTree + * @param txs - list of numberic transaction indexes associated with this address + * @param boxes - list of numberic box indexes associated with this address + * @param balanceInfo - balance information (Optional because fragments do not contain it) + */ +case class IndexedErgoAddress(treeHash: ModifierId, + txs: ListBuffer[Long], + boxes: ListBuffer[Long], + balanceInfo: Option[BalanceInfo]) extends ExtraIndex with ScorexLogging { + + override def id: ModifierId = treeHash + override def serializedId: Array[Byte] = fastIdToBytes(treeHash) + + private[extra] var boxSegmentCount: Int = 0 + private[extra] var txSegmentCount: Int = 0 + + /** + * @return total number of transactions associated with this address + */ + def txCount(): Long = segmentTreshold * txSegmentCount + txs.length + + /** + * @return total number of boxes associated with this address + */ + def boxCount(): Long = segmentTreshold * boxSegmentCount + boxes.length + + /** + * Get a range of the transactions associated with this address + * @param history - history to use + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @return array of transactions with full bodies + */ + def retrieveTxs(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoTransaction] = { + if(offset + limit > txs.length && txSegmentCount > 0) { + val range: Array[Int] = getSegmentsForRange(offset, limit) + val data: ListBuffer[Long] = ListBuffer.empty[Long] + cfor(0)(_ < range.length, _ + 1) { i => + history.typedExtraIndexById[IndexedErgoAddress](txSegmentId(treeHash, txSegmentCount - range(i))).get.txs ++=: data + } + getTxs(slice(data ++= (if(offset < txs.length) txs else Nil), offset % segmentTreshold, limit))(history) + } else + getTxs(slice(txs, offset, limit))(history) + } + + /** + * Get a range of the boxes associated with this address + * @param history - history to use + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @return array of boxes + */ + def retrieveBoxes(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoBox] = { + if(offset + limit > boxes.length && boxSegmentCount > 0) { + val range: Array[Int] = getSegmentsForRange(offset, limit) + val data: ListBuffer[Long] = ListBuffer.empty[Long] + cfor(0)(_ < range.length, _ + 1) { i => + history.typedExtraIndexById[IndexedErgoAddress](boxSegmentId(treeHash, boxSegmentCount - range(i))).get.boxes ++=: data + } + getBoxes(slice(data ++= (if(offset < boxes.length) boxes else Nil), offset % segmentTreshold, limit))(history) + } else + getBoxes(slice(boxes, offset, limit))(history) + } + + /** + * Get a range of the boxes associated with this address that are NOT spent + * @param history - history to use + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @return array of unspent boxes + */ + def retrieveUtxos(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoBox] = { + val data: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] + data ++= boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) + var segment: Int = boxSegmentCount + while(data.length < limit && segment > 0) { + segment -= 1 + history.typedExtraIndexById[IndexedErgoAddress](boxSegmentId(treeHash, segment)).get.boxes + .map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) ++=: data + } + slice(data, offset, limit).toArray + } + + /** + * Associate transaction index with this address + * @param tx - numeric transaction index + * @return this address + */ + private[extra] def addTx(tx: Long): IndexedErgoAddress = { + if(txs.lastOption.getOrElse(-1) != tx) txs += tx // check for duplicates + this + } + + /** + * Associate box with this address and update BalanceInfo + * @param iEb - box to use + * @param record - whether to add box to boxes list, used in rollbacks (true by default) + * @return this address + */ + private[extra] def addBox(iEb: IndexedErgoBox, record: Boolean = true): IndexedErgoAddress = { + if(record) boxes += iEb.globalIndex + balanceInfo.get.add(iEb.box) + this + } + + /** + * Update BalanceInfo by spending a box associated with this address + * @param box - box to spend + * @return this address + */ + private[extra] def spendBox(box: ErgoBox)(implicit ae: ErgoAddressEncoder): IndexedErgoAddress = { + balanceInfo.get.subtract(box) + this + } + + /** + * Rollback the state of this address and of boxes associted with it + * @param txTarget - remove transaction numbers above this number + * @param boxTarget - remove box numbers above this number and revert the balance + * @param _history - history handle to update address in database + */ + private[extra] def rollback(txTarget: Long, boxTarget: Long)(_history: ErgoHistory): Unit = { + + if(txs.last <= txTarget && boxes.last <= boxTarget) return + + def history: ErgoHistoryReader = _history.getReader + + val toSave: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex] + val toRemove: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId] + + // filter tx numbers + do { + val tmp = txs.takeWhile(_ <= txTarget) + txs.clear() + txs ++= tmp + if(txs.isEmpty && txSegmentCount > 0) { // entire current tx set removed, retrieving more from database if possible + val id = txSegmentId(treeHash, txSegmentCount - 1) + txs ++= history.typedExtraIndexById[IndexedErgoAddress](id).get.txs + toRemove += id + txSegmentCount -= 1 + } + }while(txCount() > 0 && txs.last > txTarget) + + // filter box numbers + do { + val tmp = boxes.takeWhile(_ <= boxTarget) + boxes.clear() + boxes ++= tmp + if(boxes.isEmpty && boxSegmentCount > 0) { // entire current box set removed, retrieving more from database if possible + val id = boxSegmentId(treeHash, boxSegmentCount - 1) + boxes ++= history.typedExtraIndexById[IndexedErgoAddress](id).get.boxes + toRemove += id + boxSegmentCount -= 1 + } + }while(boxCount() > 0 && boxes.last > boxTarget) + + if(txCount() == 0 && boxCount() == 0) + toRemove += this.id // address is empty after rollback, delete + else + toSave += this // save the changes made to this address + + _history.historyStorage.insertExtra(Array.empty, toSave.toArray) + _history.historyStorage.removeExtra(toRemove.toArray) + + } + + /** + * Create an array addresses each containing a "segmentTreshold" number of this address's transaction and box indexes. + * These special addresses have their ids calculated by "txSegmentId" and "boxSegmentId" respectively. + * @return array of addresses + */ + private[extra] def splitToSegments(): Array[IndexedErgoAddress] = { + val data: Array[IndexedErgoAddress] = new Array[IndexedErgoAddress]((txs.length / segmentTreshold) + (boxes.length / segmentTreshold)) + var i: Int = 0 + while(txs.length >= segmentTreshold) { + data(i) = new IndexedErgoAddress(txSegmentId(treeHash, txSegmentCount), txs.take(segmentTreshold), ListBuffer.empty[Long], None) + i += 1 + txSegmentCount += 1 + txs.remove(0, segmentTreshold) + } + while(boxes.length >= segmentTreshold) { + data(i) = new IndexedErgoAddress(boxSegmentId(treeHash, boxSegmentCount), ListBuffer.empty[Long], boxes.take(segmentTreshold), None) + i += 1 + boxSegmentCount += 1 + boxes.remove(0, segmentTreshold) + } + data + } +} + +object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] { + + def hashErgoTree(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) + + /** + * Calculates id of an address segment containing box indexes. + * @param hash - hash of the parent addresses ErgoTree + * @param segmentNum - numberic identifier of the segment + * @return calculated ModifierId + */ + def boxSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " box segment " + segmentNum)) + + /** + * Calculates id of an address segment transaction indexes. + * @param hash - hash of the parent addresses ErgoTree + * @param segmentNum - numberic identifier of the segment + * @return calculated ModifierId + */ + def txSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " tx segment " + segmentNum)) + + override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = { + w.putBytes(fastIdToBytes(iEa.treeHash)) + w.putUInt(iEa.txs.length) + cfor(0)(_ < iEa.txs.length, _ + 1) { i => w.putLong(iEa.txs(i))} + w.putUInt(iEa.boxes.length) + cfor(0)(_ < iEa.boxes.length, _ + 1) { i => w.putLong(iEa.boxes(i))} + w.putOption[BalanceInfo](iEa.balanceInfo)((ww, bI) => BalanceInfoSerializer.serialize(bI, ww)) + w.putInt(iEa.boxSegmentCount) + w.putInt(iEa.txSegmentCount) + } + + override def parse(r: Reader): IndexedErgoAddress = { + val addressHash: ModifierId = bytesToId(r.getBytes(32)) + val txnsLen: Long = r.getUInt() + val txns: ListBuffer[Long] = ListBuffer.empty[Long] + cfor(0)(_ < txnsLen, _ + 1) { _ => txns += r.getLong()} + val boxesLen: Long = r.getUInt() + val boxes: ListBuffer[Long] = ListBuffer.empty[Long] + cfor(0)(_ < boxesLen, _ + 1) { _ => boxes += r.getLong()} + val balanceInfo: Option[BalanceInfo] = r.getOption[BalanceInfo](BalanceInfoSerializer.parse(r)) + val iEa: IndexedErgoAddress = new IndexedErgoAddress(addressHash, txns, boxes, balanceInfo) + iEa.boxSegmentCount = r.getInt() + iEa.txSegmentCount = r.getInt() + iEa + } +} + +object IndexedErgoAddress { + + val extraIndexTypeId: ExtraIndexTypeId = 15.toByte + + val segmentTreshold: Int = 512 + + /** + * Calculate the segment offsets for the given range. + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @return array of offsets + */ + def getSegmentsForRange(offset: Int, limit: Int): Array[Int] = + (math.max(math.ceil(offset * 1F / segmentTreshold).toInt, 1) to math.ceil((offset + limit) * 1F / segmentTreshold).toInt).toArray + + /** + * Shorthand to get a range from an array by offset and limit. + * @param arr - array to get range from + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @tparam T - type of "arr" array + * @return range in "arr" array + */ + def slice[T](arr: Iterable[T], offset: Int, limit: Int): Iterable[T] = + arr.slice(arr.size - offset - limit, arr.size - offset) + + /** + * Get an array of transactions with full bodies from an array of numeric transaction indexes + * @param arr - array of numeric transaction indexes to retrieve + * @param history - database handle + * @return array of transactions with full bodies + */ + def getTxs(arr: Iterable[Long])(history: ErgoHistoryReader): Array[IndexedErgoTransaction] = // sorted to match explorer + arr.map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray.sortBy(tx => (-tx.height, tx.id)) + + /** + * Get an array of boxes from an array of numeric box indexes + * @param arr - array of numeric box indexes to retrieve + * @param history - database handle + * @return array of boxes + */ + def getBoxes(arr: Iterable[Long])(history: ErgoHistoryReader): Array[IndexedErgoBox] = + arr.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala new file mode 100644 index 0000000000..7b23ee3380 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -0,0 +1,71 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.ErgoBox +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes} +import org.ergoplatform.nodeView.wallet.WalletBox +import org.ergoplatform.wallet.Constants.ScanId +import org.ergoplatform.wallet.boxes.{ErgoBoxSerializer, TrackedBox} +import scorex.core.serialization.ScorexSerializer +import scorex.util.{ModifierId, bytesToId} +import scorex.util.serialization.{Reader, Writer} + +/** + * Index of a box. + * @param inclusionHeight - height of the block in which the creating transaction was included in + * @param spendingTxIdOpt - optional, id of the spending transaction + * @param spendingHeightOpt - optional, height of the block in which the spending transaction was included in + * @param box - underlying ErgoBox + * @param globalIndex - serial number of this output counting from genesis box + */ +class IndexedErgoBox(val inclusionHeight: Int, + var spendingTxIdOpt: Option[ModifierId], + var spendingHeightOpt: Option[Int], + val box: ErgoBox, + val globalIndex: Long) + extends WalletBox(TrackedBox(box.transactionId, + box.index, + Some(inclusionHeight), + spendingTxIdOpt, + spendingHeightOpt, + box, + Set.empty[ScanId]), + None) with ExtraIndex { + + override def serializedId: Array[Byte] = box.id + + + /** + * Fill in spending parameters. + * @param txId - id of the spending transaction + * @param txHeight - height of the block in which the spending transaction was included in + * @return this box + */ + def asSpent(txId: ModifierId, txHeight: Int): IndexedErgoBox = { + spendingTxIdOpt = Some(txId) + spendingHeightOpt = Some(txHeight) + this + } +} +object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { + + override def serialize(iEb: IndexedErgoBox, w: Writer): Unit = { + w.putInt(iEb.inclusionHeight) + w.putOption[ModifierId](iEb.spendingTxIdOpt)((ww, id) => ww.putBytes(fastIdToBytes(id))) + w.putOption[Int](iEb.spendingHeightOpt)(_.putInt(_)) + ErgoBoxSerializer.serialize(iEb.box, w) + w.putLong(iEb.globalIndex) + } + + override def parse(r: Reader): IndexedErgoBox = { + val inclusionHeight: Int = r.getInt() + val spendingTxIdOpt: Option[ModifierId] = r.getOption[ModifierId](bytesToId(r.getBytes(32))) + val spendingHeightOpt: Option[Int] = r.getOption[Int](r.getInt()) + val box: ErgoBox = ErgoBoxSerializer.parse(r) + val globalIndex: Long = r.getLong() + new IndexedErgoBox(inclusionHeight, spendingTxIdOpt, spendingHeightOpt, box, globalIndex) + } +} + +object IndexedErgoBox { + val extraIndexTypeId: ExtraIndexTypeId = 5.toByte +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala new file mode 100644 index 0000000000..c243652466 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -0,0 +1,97 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.modifiers.history.header.Header +import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.DataInput +import org.ergoplatform.modifiers.history.BlockTransactions +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes} +import scorex.core.serialization.ScorexSerializer +import scorex.util.serialization.{Reader, Writer} +import scorex.util.{ModifierId, bytesToId} +import spire.implicits.cfor + +/** + * Index of a transaction. + * @param txid - id of this transaction + * @param height - height of the block which includes this transaction + * @param globalIndex - serial number of this transaction counting from block 1 + * @param inputNums - list of transaction inputs (needed for rollback) + */ +case class IndexedErgoTransaction(txid: ModifierId, + height: Int, + globalIndex: Long, + inputNums: Array[Long]) extends ExtraIndex { + + override def id: ModifierId = txid + override def serializedId: Array[Byte] = fastIdToBytes(txid) + + private var _blockId: ModifierId = ModifierId @@ "" + private var _inclusionHeight: Int = 0 + private var _timestamp: Header.Timestamp = 0L + private var _index: Int = 0 + private var _numConfirmations: Int = 0 + private var _inputs: IndexedSeq[IndexedErgoBox] = IndexedSeq.empty[IndexedErgoBox] + private var _dataInputs: IndexedSeq[DataInput] = IndexedSeq.empty[DataInput] + private var _outputs: IndexedSeq[IndexedErgoBox] = IndexedSeq.empty[IndexedErgoBox] + private var _txSize: Int = 0 + + def blockId: ModifierId = _blockId + def inclusionHeight: Int = _inclusionHeight + def timestamp: Header.Timestamp = _timestamp + def index: Int = _index + def numConfirmations: Int = _numConfirmations + def inputs: IndexedSeq[IndexedErgoBox] = _inputs + def dataInputs: IndexedSeq[DataInput] = _dataInputs + def outputs: IndexedSeq[IndexedErgoBox] = _outputs + def txSize: Int = _txSize + + /** + * Get all information related to this transaction from database. + * @param history - database handle + * @return this transaction augmented with additional information + */ + def retrieveBody(history: ErgoHistoryReader): IndexedErgoTransaction = { + + val header: Header = history.typedModifierById[Header](history.bestHeaderIdAtHeight(height).get).get + val blockTxs: BlockTransactions = history.typedModifierById[BlockTransactions](header.transactionsId).get + + _blockId = header.id + _inclusionHeight = height + _timestamp = header.timestamp + _index = blockTxs.txs.indices.find(blockTxs.txs(_).id == txid).get + _numConfirmations = history.bestFullBlockOpt.get.height - height + _inputs = blockTxs.txs(_index).inputs.map(input => history.typedExtraIndexById[IndexedErgoBox](bytesToId(input.boxId)).get) + _dataInputs = blockTxs.txs(_index).dataInputs + _outputs = blockTxs.txs(_index).outputs.map(output => history.typedExtraIndexById[IndexedErgoBox](bytesToId(output.id)).get) + _txSize = blockTxs.txs(_index).size + + this + } +} + +object IndexedErgoTransactionSerializer extends ScorexSerializer[IndexedErgoTransaction] { + + override def serialize(iTx: IndexedErgoTransaction, w: Writer): Unit = { + w.putUByte(iTx.serializedId.length) + w.putBytes(iTx.serializedId) + w.putInt(iTx.height) + w.putLong(iTx.globalIndex) + w.putUShort(iTx.inputNums.length) + cfor(0)(_ < iTx.inputNums.length, _ + 1) { i => w.putLong(iTx.inputNums(i)) } + } + + override def parse(r: Reader): IndexedErgoTransaction = { + val idLen = r.getUByte() + val id = bytesToId(r.getBytes(idLen)) + val height = r.getInt() + val globalIndex = r.getLong() + val inputCount: Int = r.getUShort() + val inputNums: Array[Long] = Array.ofDim[Long](inputCount) + cfor(0)(_ < inputCount, _ + 1) { i => inputNums(i) = r.getLong() } + IndexedErgoTransaction(id, height, globalIndex, inputNums) + } +} + +object IndexedErgoTransaction { + val extraIndexTypeId: ExtraIndexTypeId = 10.toByte +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala new file mode 100644 index 0000000000..c0bc43d0a6 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -0,0 +1,128 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.ErgoBox +import org.ergoplatform.ErgoBox.{R4, R5, R6} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes} +import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.uniqueId +import org.ergoplatform.settings.Algos +import scorex.core.serialization.ScorexSerializer +import scorex.util.{ModifierId, bytesToId} +import scorex.util.serialization.{Reader, Writer} +import sigmastate.Values.{CollectionConstant, EvaluatedValue} +import sigmastate.{SByte, SType} + +/** + * Index of a token containing creation information. + * @param tokenId - id of this token + * @param boxId - id of the creation box + * @param amount - emission amount + * @param name - name of this token (UTF-8) + * @param description - description of this token (UTF-8) + * @param decimals - number of decimal places + */ +case class IndexedToken(tokenId: ModifierId, + boxId: ModifierId, + amount: Long, + name: String, + description: String, + decimals: Int) extends ExtraIndex { + + override def id: ModifierId = uniqueId(tokenId) + override def serializedId: Array[Byte] = fastIdToBytes(uniqueId(tokenId)) + +} + +object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { + /** + * Calculate a unique identifier for this a token. + * Necessary, because token ids are sometimes identical to box ids, which causes overwrites. + * @param tokenId - id of the token + * @return unique id for token + */ + def uniqueId(tokenId: ModifierId): ModifierId = bytesToId(Algos.hash(tokenId + "token")) + + /** + * Check if a box is creating a token. + * @param box - box to check + * @return true if the box is creation a token, false otherwise + */ + def tokenRegistersSet(box: ErgoBox): Boolean = { + + // box has tokens + if(box.additionalTokens.length == 0) return false + + // registers exist + if(!box.additionalRegisters.contains(R4) || + !box.additionalRegisters.contains(R5) || + !box.additionalRegisters.contains(R6)) + return false + + // registers correct type + try { + box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]] + box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]] + getDecimals(box.additionalRegisters(R6)) + }catch { + case _: Throwable => return false + } + + // ok + true + } + + /** + * Get the number of decimals places from a register. + * Try-catch, because some old tokens used Int to store the decimals, rather than Byte Coll + * @param reg - register to extract decimals from + * @return number of decimals places + */ + private def getDecimals(reg: EvaluatedValue[_ <: SType]): Int = { + try { + new String(reg.asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8").toInt + }catch { + case _: Throwable => reg.value.asInstanceOf[Int] + } + } + + /** + * Construct a token index from a box. Used after checking box with "tokenRegistersSet". + * @param box - box to use + * @return token index + */ + def fromBox(box: ErgoBox): IndexedToken = + IndexedToken(bytesToId(box.additionalTokens(0)._1), + bytesToId(box.id), + box.additionalTokens(0)._2, + new String(box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), + new String(box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), + getDecimals(box.additionalRegisters(R6))) + + override def serialize(iT: IndexedToken, w: Writer): Unit = { + w.putBytes(fastIdToBytes(iT.tokenId)) + w.putBytes(fastIdToBytes(iT.boxId)) + w.putULong(iT.amount) + val name: Array[Byte] = iT.name.getBytes("UTF-8") + w.putUShort(name.length) + w.putBytes(name) + val description: Array[Byte] = iT.description.getBytes("UTF-8") + w.putUShort(description.length) + w.putBytes(description) + w.putInt(iT.decimals) + } + + override def parse(r: Reader): IndexedToken = { + val tokenId: ModifierId = bytesToId(r.getBytes(32)) + val boxId: ModifierId = bytesToId(r.getBytes(32)) + val amount: Long = r.getULong() + val nameLen: Int = r.getUShort() + val name: String = new String(r.getBytes(nameLen), "UTF-8") + val descLen: Int = r.getUShort() + val description: String = new String(r.getBytes(descLen), "UTF-8") + val decimals: Int = r.getInt() + IndexedToken(tokenId, boxId, amount, name, description, decimals) + } +} + +object IndexedToken { + val extraIndexTypeId: ExtraIndexTypeId = 35.toByte +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala new file mode 100644 index 0000000000..ab519316bf --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala @@ -0,0 +1,94 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes} +import org.ergoplatform.settings.Algos +import scorex.core.serialization.ScorexSerializer +import scorex.util.{ModifierId, bytesToId} +import scorex.util.serialization.{Reader, Writer} + +/** + * Numeric index pointing to a transaction id. + * @param n - index number of a transaction + * @param m - id of a transaction + */ +case class NumericTxIndex(n: Long, m: ModifierId) extends ExtraIndex { + override def serializedId: Array[Byte] = NumericTxIndex.indexToBytes(n) +} + +object NumericTxIndexSerializer extends ScorexSerializer[NumericTxIndex] { + + override def serialize(ni: NumericTxIndex, w: Writer): Unit = { + w.putLong(ni.n) + w.putBytes(fastIdToBytes(ni.m)) + } + + override def parse(r: Reader): NumericTxIndex = { + val n: Long = r.getLong() + val m: ModifierId = bytesToId(r.getBytes(32)) + NumericTxIndex(n, m) + } +} + +object NumericTxIndex { + val extraIndexTypeId: ExtraIndexTypeId = 25.toByte + + /** + * Convert the index number of a transaction to an id for database retreival. + * @param n - index number to hash + * @return id corresponding to index number + */ + def indexToBytes(n: Long): Array[Byte] = Algos.hash("txns height " + n) + + /** + * Get a body-less transaction from database by its index number. + * @param history - database handle + * @param n - index number of a transaction + * @return transaction with given index, if found + */ + def getTxByNumber(history: ErgoHistoryReader, n: Long): Option[IndexedErgoTransaction] = + history.typedExtraIndexById[IndexedErgoTransaction](history.typedExtraIndexById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(n))).get.m) +} + +/** + * Numeric index pointing to a box id. + * @param n - index number of a box + * @param m - id of a box + */ +case class NumericBoxIndex(n: Long, m: ModifierId) extends ExtraIndex { + override def serializedId: Array[Byte] = NumericBoxIndex.indexToBytes(n) +} + +object NumericBoxIndexSerializer extends ScorexSerializer[NumericBoxIndex] { + + override def serialize(ni: NumericBoxIndex, w: Writer): Unit = { + w.putLong(ni.n) + w.putBytes(fastIdToBytes(ni.m)) + } + + override def parse(r: Reader): NumericBoxIndex = { + val n: Long = r.getLong() + val m: ModifierId = bytesToId(r.getBytes(32)) + NumericBoxIndex(n, m) + } +} + +object NumericBoxIndex { + val extraIndexTypeId: ExtraIndexTypeId = 30.toByte + + /** + * Convert the index number of a box to an id for database retreival. + * @param n - index number to hash + * @return id corresponding to index number + */ + def indexToBytes(n: Long): Array[Byte] = Algos.hash("boxes height " + n) + + /** + * Get a box from database by its index number. + * @param history - database handle + * @param n - index number of a box + * @return box with given index, if found + */ + def getBoxByNumber(history: ErgoHistoryReader, n: Long): Option[IndexedErgoBox] = + history.typedExtraIndexById[IndexedErgoBox](history.typedExtraIndexById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(n))).get.m) +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index b139f61db9..3d917e46fb 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -1,17 +1,16 @@ package org.ergoplatform.nodeView.history.storage import com.github.benmanes.caffeine.cache.Caffeine -import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.modifiers.{BlockSection, NetworkObjectTypeId} import org.ergoplatform.modifiers.history.HistoryModifierSerializer import org.ergoplatform.modifiers.history.header.Header +import org.ergoplatform.nodeView.history.extra.{ExtraIndexSerializer, ExtraIndex, IndexedErgoAddress} import org.ergoplatform.settings.{Algos, CacheSettings, ErgoSettings} -import scorex.core.ModifierTypeId import scorex.core.utils.ScorexEncoding import scorex.db.{ByteArrayWrapper, LDBFactory, LDBKVStore} import scorex.util.{ModifierId, ScorexLogging, idToBytes} -import supertagged.PostfixSugar - import scala.util.{Failure, Success, Try} +import spire.syntax.all.cfor /** * Storage for Ergo history @@ -19,9 +18,10 @@ import scala.util.{Failure, Success, Try} * @param indexStore - Additional key-value storage for indexes, required by History for efficient work. * contains links to bestHeader, bestFullBlock, heights and scores for different blocks, etc. * @param objectsStore - key-value store, where key is id of ErgoPersistentModifier and value is it's bytes + * @param extraStore - key-value store, where key is id of Index and value is it's bytes * @param config - cache configs */ -class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, config: CacheSettings) +class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStore: LDBKVStore, config: CacheSettings) extends ScorexLogging with AutoCloseable with ScorexEncoding { @@ -36,6 +36,11 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c .maximumSize(config.history.blockSectionsCacheSize) .build[String, BlockSection]() + private val extraCache = + Caffeine.newBuilder() + .maximumSize(config.history.extraCacheSize) + .build[String, ExtraIndex]() + private val indexCache = Caffeine.newBuilder() .maximumSize(config.history.indexesCacheSize) @@ -52,29 +57,48 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c private def removeModifier(id: ModifierId): Unit = { headersCache.invalidate(id) blockSectionsCache.invalidate(id) + extraCache.invalidate(id) } def modifierBytesById(id: ModifierId): Option[Array[Byte]] = { - objectsStore.get(idToBytes(id)).map(_.tail) // removing modifier type byte with .tail + objectsStore.get(idToBytes(id)).map(_.tail).orElse(extraStore.get(idToBytes(id))) // removing modifier type byte with .tail (only in objectsStore) } - def modifierTypeAndBytesById(id: ModifierId): Option[(ModifierTypeId, Array[Byte])] = { - objectsStore.get(idToBytes(id)).map(bs => (bs.head @@ ModifierTypeId, bs.tail)) // first byte is type id, tail is modifier bytes + /** + * @return bytes and type of a network object stored in the database with identifier `id` + */ + def modifierTypeAndBytesById(id: ModifierId): Option[(NetworkObjectTypeId.Value, Array[Byte])] = { + objectsStore.get(idToBytes(id)).map(bs => (NetworkObjectTypeId.fromByte(bs.head), bs.tail)) // first byte is type id, tail is modifier bytes } def modifierById(id: ModifierId): Option[BlockSection] = - lookupModifier(id) orElse - objectsStore.get(idToBytes(id)).flatMap { bytes => - HistoryModifierSerializer.parseBytesTry(bytes) match { - case Success(pm) => - log.trace(s"Cache miss for existing modifier $id") - cacheModifier(pm) - Some(pm) - case Failure(_) => - log.warn(s"Failed to parse modifier ${encoder.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") - None - } + lookupModifier(id) orElse objectsStore.get(idToBytes(id)).flatMap { bytes => + HistoryModifierSerializer.parseBytesTry(bytes) match { + case Success(pm) => + log.trace(s"Cache miss for existing modifier $id") + cacheModifier(pm) + Some(pm) + case Failure(_) => + log.warn(s"Failed to parse modifier ${encoder.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") + None + } + } + + def getExtraIndex(id: ModifierId): Option[ExtraIndex] = { + Option(extraCache.getIfPresent(id)) orElse extraStore.get(idToBytes(id)).flatMap { bytes => + ExtraIndexSerializer.parseBytesTry(bytes) match { + case Success(pm) => + log.trace(s"Cache miss for existing index $id") + if(pm.isInstanceOf[IndexedErgoAddress]){ + extraCache.put(pm.id, pm) // only cache addresses + } + Some(pm) + case Failure(_) => + log.warn(s"Failed to parse index ${encoder.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") + None } + } + } def getIndex(id: ByteArrayWrapper): Option[Array[Byte]] = Option(indexCache.getIfPresent(id)).orElse { @@ -84,25 +108,37 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c } } - def get(id: ModifierId): Option[Array[Byte]] = objectsStore.get(idToBytes(id)) + def get(id: ModifierId): Option[Array[Byte]] = objectsStore.get(idToBytes(id)).orElse(extraStore.get(idToBytes(id))) - def contains(id: ModifierId): Boolean = objectsStore.get(idToBytes(id)).isDefined + def contains(id: ModifierId): Boolean = get(id).isDefined - def insert(indexesToInsert: Seq[(ByteArrayWrapper, Array[Byte])], - objectsToInsert: Seq[BlockSection]): Try[Unit] = { + def insert(indexesToInsert: Array[(ByteArrayWrapper, Array[Byte])], + objectsToInsert: Array[BlockSection]): Try[Unit] = { objectsStore.insert( - objectsToInsert.map(m => idToBytes(m.id) -> HistoryModifierSerializer.toBytes(m)) + objectsToInsert.map(mod => (mod.serializedId, HistoryModifierSerializer.toBytes(mod))) ).flatMap { _ => - objectsToInsert.foreach(o => cacheModifier(o)) + cfor(0)(_ < objectsToInsert.length, _ + 1) { i => cacheModifier(objectsToInsert(i))} if (indexesToInsert.nonEmpty) { indexStore.insert(indexesToInsert.map { case (k, v) => k.data -> v }).map { _ => - indexesToInsert.foreach(kv => indexCache.put(kv._1, kv._2)) + cfor(0)(_ < indexesToInsert.length, _ + 1) { i => indexCache.put(indexesToInsert(i)._1, indexesToInsert(i)._2)} () } } else Success(()) } } + def insertExtra(indexesToInsert: Array[(Array[Byte], Array[Byte])], + objectsToInsert: Array[ExtraIndex]): Unit = { + extraStore.insert(objectsToInsert.map(mod => (mod.serializedId, ExtraIndexSerializer.toBytes(mod)))) + cfor(0)(_ < objectsToInsert.length, _ + 1) { i => val ei = objectsToInsert(i); extraCache.put(ei.id, ei)} + cfor(0)(_ < indexesToInsert.length, _ + 1) { i => extraStore.insert(indexesToInsert(i)._1, indexesToInsert(i)._2)} + } + + def removeExtra(indexesToRemove: Array[ModifierId]) : Unit = { + extraStore.remove(indexesToRemove.map(idToBytes)) + cfor(0)(_ < indexesToRemove.length, _ + 1) { i => removeModifier(indexesToRemove(i)) } + } + /** * Insert single object to database. This version allows for efficient insert * when identifier and bytes of object (i.e. modifier, a block section) are known. @@ -123,17 +159,13 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c * @param idsToRemove - identifiers of modifiers to remove * @return */ - def remove(indicesToRemove: Seq[ByteArrayWrapper], - idsToRemove: Seq[ModifierId]): Try[Unit] = { + def remove(indicesToRemove: Array[ByteArrayWrapper], + idsToRemove: Array[ModifierId]): Try[Unit] = { objectsStore.remove(idsToRemove.map(idToBytes)).map { _ => - idsToRemove.foreach { id => - removeModifier(id) - } + cfor(0)(_ < idsToRemove.length, _ + 1) { i => removeModifier(idsToRemove(i))} indexStore.remove(indicesToRemove.map(_.data)).map { _ => - indicesToRemove.foreach { id => - indexCache.invalidate(id) - } + cfor(0)(_ < indicesToRemove.length, _ + 1) { i => indexCache.invalidate(indicesToRemove(i))} () } } @@ -141,6 +173,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c override def close(): Unit = { log.warn("Closing history storage...") + extraStore.close() indexStore.close() objectsStore.close() } @@ -151,6 +184,7 @@ object HistoryStorage { def apply(ergoSettings: ErgoSettings): HistoryStorage = { val indexStore = LDBFactory.createKvDb(s"${ergoSettings.directory}/history/index") val objectsStore = LDBFactory.createKvDb(s"${ergoSettings.directory}/history/objects") - new HistoryStorage(indexStore, objectsStore, ergoSettings.cacheSettings) + val extraStore = LDBFactory.createKvDb(s"${ergoSettings.directory}/history/extra") + new HistoryStorage(indexStore, objectsStore, extraStore, ergoSettings.cacheSettings) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala index c17816d69b..b86cd81f3d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala @@ -11,7 +11,6 @@ import scorex.util.{ModifierId, bytesToId, idToBytes} import scala.annotation.tailrec import scala.collection.immutable.TreeMap -import scala.collection.mutable import scala.util.{Failure, Success, Try} /** @@ -135,7 +134,7 @@ trait FullBlockProcessor extends HeadersProcessor { } //Orphaned block or full chain is not initialized yet logStatus(Seq(), Seq(), params.fullBlock, None) - historyStorage.insert(Seq.empty, Seq(params.newModRow)).map { _ => + historyStorage.insert(Array.empty[(ByteArrayWrapper, Array[Byte])], Array(params.newModRow)).map { _ => ProgressInfo(None, Seq.empty, Seq.empty, Seq.empty) } } @@ -230,17 +229,17 @@ trait FullBlockProcessor extends HeadersProcessor { } private def pruneBlockDataAt(heights: Seq[Int]): Try[Unit] = { - val toRemove: Seq[ModifierId] = heights.flatMap(h => headerIdsAtHeight(h)) + val toRemove: Array[ModifierId] = heights.flatMap(h => headerIdsAtHeight(h)) .flatMap(id => typedModifierById[Header](id)) - .flatMap(_.sectionIds.map(_._2)) - historyStorage.remove(mutable.WrappedArray.empty, toRemove) + .flatMap(_.sectionIds.map(_._2)).toArray + historyStorage.remove(Array.empty, toRemove) } private def updateStorage(newModRow: BlockSection, bestFullHeaderId: ModifierId, additionalIndexes: Seq[(ByteArrayWrapper, Array[Byte])]): Try[Unit] = { - val indicesToInsert = Seq(BestFullBlockKey -> idToBytes(bestFullHeaderId)) ++ additionalIndexes - historyStorage.insert(indicesToInsert, Seq(newModRow)).flatMap { _ => + val indicesToInsert = Array(BestFullBlockKey -> idToBytes(bestFullHeaderId)) ++ additionalIndexes + historyStorage.insert(indicesToInsert, Array(newModRow)).flatMap { _ => if (headersHeight >= fullBlockHeight) Success(()) else diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala index 9e207d8a12..05b344be92 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala @@ -3,12 +3,13 @@ package org.ergoplatform.nodeView.history.storage.modifierprocessors import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.modifiers.{NonHeaderBlockSection, ErgoFullBlock, BlockSection} +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NonHeaderBlockSection} import org.ergoplatform.settings.ValidationRules._ import org.ergoplatform.settings.{Algos, ErgoValidationSettings} import scorex.core.consensus.ProgressInfo import scorex.core.utils.ScorexEncoding import scorex.core.validation.{ModifierValidator, _} +import scorex.db.ByteArrayWrapper import scorex.util.ModifierId import scala.reflect.ClassTag @@ -82,7 +83,7 @@ trait FullBlockSectionProcessor extends BlockSectionProcessor with FullBlockProc } private def justPutToHistory(m: NonHeaderBlockSection): Try[ProgressInfo[BlockSection]] = { - historyStorage.insert(Seq.empty, Seq(m)).map { _ => + historyStorage.insert(Array.empty[(ByteArrayWrapper, Array[Byte])], Array[BlockSection](m)).map { _ => ProgressInfo(None, Seq.empty, Seq.empty, Seq.empty) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala index 36010fc66d..5216f06662 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala @@ -89,7 +89,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score * @return ProgressInfo - info required for State to be consistent with History */ protected def process(h: Header): Try[ProgressInfo[BlockSection]] = synchronized { - val dataToInsert: (Seq[(ByteArrayWrapper, Array[Byte])], Seq[BlockSection]) = toInsert(h) + val dataToInsert: (Array[(ByteArrayWrapper, Array[Byte])], Array[BlockSection]) = toInsert(h) historyStorage.insert(dataToInsert._1, dataToInsert._2).flatMap { _ => bestHeaderIdOpt match { @@ -107,7 +107,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score /** * Data to add to and remove from the storage to process this modifier */ - private def toInsert(h: Header): (Seq[(ByteArrayWrapper, Array[Byte])], Seq[BlockSection]) = { + private def toInsert(h: Header): (Array[(ByteArrayWrapper, Array[Byte])], Array[BlockSection]) = { val requiredDifficulty: Difficulty = h.requiredDifficulty val score = scoreOf(h.parentId).getOrElse(BigInt(0)) + requiredDifficulty val bestRow: Seq[(ByteArrayWrapper, Array[Byte])] = @@ -121,7 +121,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score orphanedBlockHeaderIdsRow(h, score) } - (Seq(scoreRow, heightRow) ++ bestRow ++ headerIdsRow, Seq(h)) + (Array(scoreRow, heightRow) ++ bestRow ++ headerIdsRow, Array(h)) } /** @@ -156,7 +156,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score * * @return Success() if header is valid, Failure(error) otherwise */ - protected def validate(header: Header): Try[Unit] = new HeaderValidator().validate(header).toTry + protected def validate(header: Header): Try[Unit] = HeaderValidator.validate(header).toTry protected val BestHeaderKey: ByteArrayWrapper = ByteArrayWrapper(Array.fill(HashLength)(Header.modifierTypeId)) @@ -307,10 +307,12 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score } } - class HeaderValidator extends ScorexEncoding { + private object HeaderValidator extends ScorexEncoding { private def validationState: ValidationState[Unit] = ModifierValidator(ErgoValidationSettings.initial) + private def time(): ErgoHistory.Time = System.currentTimeMillis() + def validate(header: Header): ValidationResult[Unit] = { if (header.isGenesis) { validateGenesisBlockHeader(header) @@ -333,7 +335,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score .validateEquals(hdrRequiredDifficulty, header.requiredDifficulty, chainSettings.initialDifficulty, header.id, header.modifierTypeId) .validateNot(alreadyApplied, historyStorage.contains(header.id), InvalidModifier(header.toString, header.id, header.modifierTypeId)) .validate(hdrTooOld, fullBlockHeight < nodeSettings.keepVersions, InvalidModifier(heightOf(header.parentId).toString, header.id, header.modifierTypeId)) - .validate(hdrFutureTimestamp, header.timestamp - timeProvider.time() <= MaxTimeDrift, InvalidModifier(s"${header.timestamp} vs ${timeProvider.time()}", header.id, header.modifierTypeId)) + .validate(hdrFutureTimestamp, header.timestamp - time() <= MaxTimeDrift, InvalidModifier(s"${header.timestamp} vs ${time()}", header.id, header.modifierTypeId)) .result } @@ -348,7 +350,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score .validateEquals(hdrRequiredDifficulty, header.requiredDifficulty, requiredDifficultyAfter(parent), header.id, header.modifierTypeId) .validate(hdrTooOld, heightOf(header.parentId).exists(h => fullBlockHeight - h < nodeSettings.keepVersions), InvalidModifier(heightOf(header.parentId).toString, header.id, header.modifierTypeId)) .validateSemantics(hdrParentSemantics, isSemanticallyValid(header.parentId), InvalidModifier(s"Parent semantics broken", header.id, header.modifierTypeId)) - .validate(hdrFutureTimestamp, header.timestamp - timeProvider.time() <= MaxTimeDrift, InvalidModifier(s"${header.timestamp} vs ${timeProvider.time()}", header.id, header.modifierTypeId)) + .validate(hdrFutureTimestamp, header.timestamp - time() <= MaxTimeDrift, InvalidModifier(s"${header.timestamp} vs ${time()}", header.id, header.modifierTypeId)) .validateNot(alreadyApplied, historyStorage.contains(header.id), InvalidModifier(s"${header.id} already applied", header.id, header.modifierTypeId)) .validate(hdrCheckpoint, checkpointCondition(header), InvalidModifier(s"${header.id} wrong checkpoint", header.id, header.modifierTypeId)) .result diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala index 97af3cc922..508e203c18 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala @@ -1,11 +1,9 @@ package org.ergoplatform.nodeView.history.storage.modifierprocessors -import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.modifiers.{ErgoFullBlock, NetworkObjectTypeId} import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.settings.{ChainSettings, ErgoSettings, NodeConfigurationSettings} -import scorex.core.ModifierTypeId -import scorex.core.utils.NetworkTimeProvider import scorex.util.{ModifierId, ScorexLogging} import scala.annotation.tailrec @@ -16,8 +14,6 @@ import scala.annotation.tailrec trait ToDownloadProcessor extends BasicReaders with ScorexLogging { import ToDownloadProcessor._ - protected val timeProvider: NetworkTimeProvider - protected val settings: ErgoSettings // A node is considering that the chain is synced if sees a block header with timestamp no more @@ -42,12 +38,12 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { /** * Get modifier ids to download to synchronize full blocks * @param howManyPerType how many ModifierIds per ModifierTypeId to fetch - * @param condition filter only ModifierIds that pass this condition + * @param condition only ModifierIds which pass filter are included into results * @return next max howManyPerType ModifierIds by ModifierTypeId to download filtered by condition */ def nextModifiersToDownload(howManyPerType: Int, estimatedTip: Option[Int], - condition: (ModifierTypeId, ModifierId) => Boolean): Map[ModifierTypeId, Seq[ModifierId]] = { + condition: (NetworkObjectTypeId.Value, ModifierId) => Boolean): Map[NetworkObjectTypeId.Value, Seq[ModifierId]] = { val FullBlocksToDownloadAhead = 192 // how many full blocks to download forwards during active sync @@ -56,8 +52,8 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { @tailrec def continuation(height: Int, - acc: Map[ModifierTypeId, Vector[ModifierId]], - maxHeight: Int = Int.MaxValue): Map[ModifierTypeId, Vector[ModifierId]] = { + acc: Map[NetworkObjectTypeId.Value, Vector[ModifierId]], + maxHeight: Int = Int.MaxValue): Map[NetworkObjectTypeId.Value, Vector[ModifierId]] = { if (height > maxHeight) { acc } else { @@ -81,7 +77,7 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { bestFullBlockOpt match { case _ if !isHeadersChainSynced || !nodeSettings.verifyTransactions => - // do not download full blocks if no headers-chain synced yet and suffix enabled or SPV mode + // do not download full blocks if no headers-chain synced yet or SPV mode Map.empty case Some(fb) if farAwayFromBeingSynced(fb) => // when far away from blockchain tip @@ -100,14 +96,14 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { /** * Checks whether it's time to download full chain, and returns toDownload modifiers */ - protected def toDownload(header: Header): Seq[(ModifierTypeId, ModifierId)] = { + protected def toDownload(header: Header): Seq[(NetworkObjectTypeId.Value, ModifierId)] = { if (!nodeSettings.verifyTransactions) { // A regime that do not download and verify transaction Nil } else if (pruningProcessor.shouldDownloadBlockAtHeight(header.height)) { // Already synced and header is not too far back. Download required modifiers. requiredModifiersForHeader(header) - } else if (!isHeadersChainSynced && header.isNew(timeProvider, chainSettings.blockInterval * headerChainDiff)) { + } else if (!isHeadersChainSynced && header.isNew(chainSettings.blockInterval * headerChainDiff)) { // Headers chain is synced after this header. Start downloading full blocks pruningProcessor.updateBestFullBlock(header) log.info(s"Headers chain is likely synced after header ${header.encodedId} at height ${header.height}") @@ -117,11 +113,14 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { } } - def requiredModifiersForHeader(h: Header): Seq[(ModifierTypeId, ModifierId)] = { + /** + * @return block sections needed to be downloaded after header `h` , and defined by the header + */ + def requiredModifiersForHeader(h: Header): Seq[(NetworkObjectTypeId.Value, ModifierId)] = { if (!nodeSettings.verifyTransactions) { - Nil + Nil // no block sections to be downloaded in SPV mode } else if (nodeSettings.stateType.requireProofs) { - h.sectionIds + h.sectionIds // download block transactions, extension and UTXO set transformations proofs in "digest" mode } else { h.sectionIdsWithNoProof // do not download UTXO set transformation proofs if UTXO set is stored } diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index f3d6c1df6d..8e5431c369 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -88,17 +88,13 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, /** * Method to put a transaction into the memory pool. Validation of the transactions against - * the state is done in NodeVieHolder. This put() method can check whether a transaction is valid + * the state is done in NodeViewHolder. This put() method can check whether a transaction is valid * @param unconfirmedTx * @return Success(updatedPool), if transaction successfully added to the pool, Failure(_) otherwise */ def put(unconfirmedTx: UnconfirmedTransaction): ErgoMemPool = { - if (!pool.contains(unconfirmedTx.id)) { - val updatedPool = pool.put(unconfirmedTx, feeFactor(unconfirmedTx)) - new ErgoMemPool(updatedPool, stats, sortingOption) - } else { - this - } + val updatedPool = pool.put(unconfirmedTx, feeFactor(unconfirmedTx)) + new ErgoMemPool(updatedPool, stats, sortingOption) } def put(txs: TraversableOnce[UnconfirmedTransaction]): ErgoMemPool = { @@ -137,8 +133,14 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, pool.get(unconfirmedTransactionId) match { case Some(utx) => invalidate(utx) case None => - log.warn(s"Can't invalidate transaction $unconfirmedTransactionId as it is not in the pool") - this + log.warn(s"pool.get failed for $unconfirmedTransactionId") + pool.orderedTransactions.valuesIterator.find(_.id == unconfirmedTransactionId) match { + case Some(utx) => + invalidate(utx) + case None => + log.warn(s"Can't invalidate transaction $unconfirmedTransactionId as it is not in the pool") + this + } } } diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala index 30b92bcdfc..127edd8201 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala @@ -17,12 +17,12 @@ import scala.collection.immutable.TreeMap * @param outputs - mapping `box.id` -> `WeightedTxId(tx.id,tx.weight)` required for getting a transaction by its output box * @param inputs - mapping `box.id` -> `WeightedTxId(tx.id,tx.weight)` required for getting a transaction by its input box id */ -case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTransaction], - transactionsRegistry: TreeMap[ModifierId, WeightedTxId], - invalidatedTxIds: ApproximateCacheLike[String], - outputs: TreeMap[BoxId, WeightedTxId], - inputs: TreeMap[BoxId, WeightedTxId]) - (implicit settings: ErgoSettings) extends ScorexLogging { +class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTransaction], + val transactionsRegistry: TreeMap[ModifierId, WeightedTxId], + val invalidatedTxIds: ApproximateCacheLike[String], + val outputs: TreeMap[BoxId, WeightedTxId], + val inputs: TreeMap[BoxId, WeightedTxId]) + (implicit settings: ErgoSettings) extends ScorexLogging { import OrderedTxPool.weighted @@ -66,14 +66,26 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT */ def put(unconfirmedTx: UnconfirmedTransaction, feeFactor: Int): OrderedTxPool = { val tx = unconfirmedTx.transaction - val wtx = weighted(tx, feeFactor) - val newPool = OrderedTxPool( - orderedTransactions.updated(wtx, unconfirmedTx), - transactionsRegistry.updated(wtx.id, wtx), - invalidatedTxIds, - outputs ++ tx.outputs.map(_.id -> wtx), - inputs ++ tx.inputs.map(_.boxId -> wtx) - ).updateFamily(tx, wtx.weight, System.currentTimeMillis(), 0) + + val newPool = transactionsRegistry.get(tx.id) match { + case Some(wtx) => + new OrderedTxPool( + orderedTransactions.updated(wtx, unconfirmedTx), + transactionsRegistry, + invalidatedTxIds, + outputs, + inputs + ) + case None => + val wtx = weighted(tx, feeFactor) + new OrderedTxPool( + orderedTransactions.updated(wtx, unconfirmedTx), + transactionsRegistry.updated(wtx.id, wtx), + invalidatedTxIds, + outputs ++ tx.outputs.map(_.id -> wtx), + inputs ++ tx.inputs.map(_.boxId -> wtx) + ).updateFamily(tx, wtx.weight, System.currentTimeMillis(), 0) + } if (newPool.orderedTransactions.size > mempoolCapacity) { val victim = newPool.orderedTransactions.last._2 newPool.remove(victim) @@ -94,7 +106,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT def remove(tx: ErgoTransaction): OrderedTxPool = { transactionsRegistry.get(tx.id) match { case Some(wtx) => - OrderedTxPool( + new OrderedTxPool( orderedTransactions - wtx, transactionsRegistry - tx.id, invalidatedTxIds, @@ -107,11 +119,14 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT def remove(utx: UnconfirmedTransaction): OrderedTxPool = remove(utx.transaction) + /** + * Remove transaction from the pool and add it to invalidated transaction ids cache + */ def invalidate(unconfirmedTx: UnconfirmedTransaction): OrderedTxPool = { val tx = unconfirmedTx.transaction transactionsRegistry.get(tx.id) match { case Some(wtx) => - OrderedTxPool( + new OrderedTxPool( orderedTransactions - wtx, transactionsRegistry - tx.id, invalidatedTxIds.put(tx.id), @@ -119,17 +134,20 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT inputs -- tx.inputs.map(_.boxId) ).updateFamily(tx, -wtx.weight, System.currentTimeMillis(), depth = 0) case None => - OrderedTxPool(orderedTransactions, transactionsRegistry, invalidatedTxIds.put(tx.id), outputs, inputs) + if (orderedTransactions.valuesIterator.exists(utx => utx.id == tx.id)) { + new OrderedTxPool( + orderedTransactions.filter(_._2.id != tx.id), + transactionsRegistry - tx.id, + invalidatedTxIds.put(tx.id), + outputs -- tx.outputs.map(_.id), + inputs -- tx.inputs.map(_.boxId) + ) + } else { + new OrderedTxPool(orderedTransactions, transactionsRegistry, invalidatedTxIds.put(tx.id), outputs, inputs) + } } } - def filter(condition: UnconfirmedTransaction => Boolean): OrderedTxPool = { - orderedTransactions.foldLeft(this)((pool, entry) => { - val tx = entry._2 - if (condition(tx)) pool else pool.remove(tx) - }) - } - /** * Do not place transaction in the pool if the transaction known to be invalid, pool already has it, or the pool * is overfull. @@ -175,13 +193,14 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT this } else { - val uniqueTxIds: Set[WeightedTxId] = tx.inputs.flatMap(input => this.outputs.get(input.boxId))(collection.breakOut) + val uniqueTxIds: Set[WeightedTxId] = tx.inputs.flatMap(input => this.outputs.get(input.boxId)).toSet val parentTxs = uniqueTxIds.flatMap(wtx => this.orderedTransactions.get(wtx).map(ut => wtx -> ut)) parentTxs.foldLeft(this) { case (pool, (wtx, ut)) => val parent = ut.transaction val newWtx = WeightedTxId(wtx.id, wtx.weight + weight, wtx.feePerFactor, wtx.created) - val newPool = OrderedTxPool(pool.orderedTransactions - wtx + (newWtx -> ut), + val newPool = new OrderedTxPool( + pool.orderedTransactions - wtx + (newWtx -> ut), pool.transactionsRegistry.updated(parent.id, newWtx), invalidatedTxIds, parent.outputs.foldLeft(pool.outputs)((newOutputs, box) => newOutputs.updated(box.id, newWtx)), @@ -220,7 +239,7 @@ object OrderedTxPool { val cacheSettings = settings.cacheSettings.mempool val frontCacheSize = cacheSettings.invalidModifiersCacheSize val frontCacheExpiration = cacheSettings.invalidModifiersCacheExpiration - OrderedTxPool( + new OrderedTxPool( TreeMap.empty[WeightedTxId, UnconfirmedTransaction], TreeMap.empty[ModifierId, WeightedTxId], ExpiringApproximateCache.empty(frontCacheSize, frontCacheExpiration), diff --git a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala index a58e55b445..d4e1f27595 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala @@ -26,7 +26,7 @@ import scala.util.{Failure, Success, Try} * See https://eprint.iacr.org/2016/994 for details on this mode. */ class DigestState protected(override val version: VersionTag, - override val rootHash: ADDigest, + override val rootDigest: ADDigest, override val store: LDBVersionedStore, ergoSettings: ErgoSettings) extends ErgoState[DigestState] @@ -48,7 +48,7 @@ class DigestState protected(override val version: VersionTag, val knownBoxesTry = ErgoState.stateChanges(transactions).map { stateChanges => val boxesFromProofs: Seq[ErgoBox] = - proofs.verify(stateChanges, rootHash, expectedHash).get.map(v => ErgoBoxSerializer.parseBytes(v)) + proofs.verify(stateChanges, rootDigest, expectedHash).get.map(v => ErgoBoxSerializer.parseBytes(v)) (transactions.flatMap(_.outputs) ++ boxesFromProofs).map(o => (ByteArrayWrapper(o.id), o)).toMap } @@ -107,7 +107,7 @@ class DigestState protected(override val version: VersionTag, private def processFullBlock: ModifierProcessing[DigestState] = { case fb: ErgoFullBlock if nodeSettings.verifyTransactions => log.info(s"Got new full block ${fb.encodedId} at height ${fb.header.height} with root " + - s"${Algos.encode(fb.header.stateRoot)}. Our root is ${Algos.encode(rootHash)}") + s"${Algos.encode(fb.header.stateRoot)}. Our root is ${Algos.encode(rootDigest)}") validate(fb) .flatMap { _ => val version: VersionTag = idToVersion(fb.header.id) diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala index 327cca2242..cfc95bba4b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala @@ -64,6 +64,14 @@ trait ErgoState[IState <: ErgoState[IState]] extends ErgoStateReader { */ def getReader: ErgoStateReader = this + /** + * Close database where state-related data lives + */ + def closeStorage(): Unit = { + log.warn("Closing state's store.") + store.close() + } + } object ErgoState extends ScorexLogging { @@ -262,8 +270,8 @@ object ErgoState extends ScorexLogging { val bh = BoxHolder(boxes) UtxoState.fromBoxHolder(bh, boxes.headOption, stateDir, constants, LaunchParameters).ensuring(us => { - log.info(s"Genesis UTXO state generated with hex digest ${Base16.encode(us.rootHash)}") - java.util.Arrays.equals(us.rootHash, constants.settings.chainSettings.genesisStateDigest) && us.version == genesisStateVersion + log.info(s"Genesis UTXO state generated with hex digest ${Base16.encode(us.rootDigest)}") + java.util.Arrays.equals(us.rootDigest, constants.settings.chainSettings.genesisStateDigest) && us.version == genesisStateVersion }) -> bh } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala index 7da217fc7c..9479fe4cf5 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala @@ -8,9 +8,22 @@ import scorex.crypto.hash.Digest32 import scorex.db.LDBVersionedStore import scorex.util.ScorexLogging +/** + * State-related data and functions related to any state implementation ("utxo" or "digest") which are + * not modifying the state (so only reading it) + */ trait ErgoStateReader extends NodeViewComponent with ScorexLogging { - def rootHash: ADDigest + /** + * Root hash and height of AVL+ tree authenticating UTXO set + */ + def rootDigest: ADDigest + + /** + * Current version of the state + * Must be ID of last block applied + */ + def version: VersionTag val store: LDBVersionedStore val constants: StateConstants @@ -19,6 +32,13 @@ trait ErgoStateReader extends NodeViewComponent with ScorexLogging { protected lazy val votingSettings: VotingSettings = chainSettings.voting + /** + * If the state is in its genesis version (before genesis block) + */ + def isGenesis: Boolean = { + rootDigest.sameElements(constants.settings.chainSettings.genesisStateDigest) + } + def stateContext: ErgoStateContext = ErgoStateReader.storageStateContext(store, constants) /** @@ -28,14 +48,6 @@ trait ErgoStateReader extends NodeViewComponent with ScorexLogging { def genesisBoxes: Seq[ErgoBox] = ErgoState.genesisBoxes(chainSettings) - //must be ID of last applied modifier - def version: VersionTag - - def closeStorage(): Unit = { - log.warn("Closing state's store.") - store.close() - } - } object ErgoStateReader extends ScorexLogging { diff --git a/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala new file mode 100644 index 0000000000..6a57e76c50 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala @@ -0,0 +1,36 @@ +package org.ergoplatform.nodeView.state + +import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.nodeView.state.UtxoState.ManifestId + +/** + * Container for UTXO set snapshots the node holds + * @param availableManifests - available UTXO set snapshot manifests and corresponding heights + */ +class SnapshotsInfo(val availableManifests: Map[Height, ManifestId]) { + + /** + * @return new container instance with new snapshot added + */ + def withNewManifest(height: Height, manifestId: ManifestId): SnapshotsInfo = { + new SnapshotsInfo(availableManifests.updated(height, manifestId)) + } + + /** + * @return whether snapshots available + */ + def nonEmpty: Boolean = availableManifests.nonEmpty +} + +object SnapshotsInfo { + + /** + * @return empty container with no snapshots + */ + val empty: SnapshotsInfo = new SnapshotsInfo(Map.empty) + +} + + + + diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala index b585c0bada..4265a6dfda 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala @@ -13,11 +13,11 @@ import org.ergoplatform.settings.{Algos, Parameters} import org.ergoplatform.utils.LoggingUtil import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import scorex.core._ -import scorex.core.transaction.Transaction import scorex.core.transaction.state.TransactionValidation import scorex.core.utils.ScorexEncoding import scorex.core.validation.{ModifierValidator} import scorex.crypto.authds.avltree.batch._ +import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, BatchAVLProverSubtree} import scorex.crypto.authds.{ADDigest, ADValue} import scorex.crypto.hash.Digest32 import scorex.db.{ByteArrayWrapper, LDBVersionedStore} @@ -42,12 +42,12 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 with UtxoStateReader with ScorexEncoding { - override def rootHash: ADDigest = persistentProver.synchronized { + import UtxoState.metadata + + override def rootDigest: ADDigest = persistentProver.synchronized { persistentProver.digest } - import UtxoState.metadata - override def rollbackTo(version: VersionTag): Try[UtxoState] = persistentProver.synchronized { val p = persistentProver log.info(s"Rollback UtxoState to version ${Algos.encoder.encode(version)}") @@ -101,7 +101,7 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 opsResult } ModifierValidator(stateContext.validationSettings) - .validateNoFailure(fbOperationFailed, blockOpsTry, Transaction.ModifierTypeId) + .validateNoFailure(fbOperationFailed, blockOpsTry, ErgoTransaction.modifierTypeId) .validateEquals(fbDigestIncorrect, expectedDigest, persistentProver.digest, headerId, Header.modifierTypeId) .result .toTry @@ -132,7 +132,7 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 log.debug(s"Trying to apply full block with header ${fb.header.encodedId} at height $height") - val inRoot = rootHash + val inRoot = rootDigest val stateTry = stateContext.appendFullBlock(fb).flatMap { newStateContext => val txsTry = applyTransactions(fb.blockTransactions.txs, fb.header.id, fb.header.stateRoot, newStateContext) @@ -236,6 +236,27 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 object UtxoState { + /** + * Short synonym for AVL+ tree type used in the node + */ + type Manifest = BatchAVLProverManifest[Digest32] + + /** + * Short synonym for AVL subtree type used in the node + */ + type Subtree = BatchAVLProverSubtree[Digest32] + + + /** + * Manifest is associated with 32 bytes cryptographically strong unique id (root hash of the AVL tree under manifest) + */ + type ManifestId = Digest32 + + /** + * Subtree is associated with 32 bytes cryptographically strong unique id (hash of subtree's root node) + */ + type SubtreeId = Digest32 + private lazy val bestVersionKey = Algos.hash("best state version") val EmissionBoxIdKey: Digest32 = Algos.hash("emission box id key") diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala index abdfbe04b9..fe1cbb237b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala @@ -48,7 +48,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco /** * Remove pre-3.3.0 derivation paths */ - def removePaths(): Try[Unit] = store.remove(Seq(SecretPathsKey)) + def removePaths(): Try[Unit] = store.remove(Array(SecretPathsKey)) /** * Store wallet-related public key in the database @@ -59,7 +59,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco store.insert { publicKeys.map { publicKey => pubKeyPrefixKey(publicKey) -> ExtendedPublicKeySerializer.toBytes(publicKey) - } + }.toArray } } @@ -99,7 +99,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco * @param ctx - state context */ def updateStateContext(ctx: ErgoStateContext): Try[Unit] = store - .insert(Seq(StateContextKey -> ctx.bytes)) + .insert(Array(StateContextKey -> ctx.bytes)) /** * Read state context from the database @@ -116,7 +116,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco */ def updateChangeAddress(address: P2PKAddress): Try[Unit] = { val bytes = settings.chainSettings.addressEncoder.toString(address).getBytes(Constants.StringEncoding) - store.insert(Seq(ChangeAddressKey -> bytes)) + store.insert(Array(ChangeAddressKey -> bytes)) } /** @@ -139,7 +139,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco def addScan(scanReq: ScanRequest): Try[Scan] = { val id = ScanId @@ (lastUsedScanId + 1).toShort scanReq.toScan(id).flatMap { app => - store.insert(Seq( + store.insert(Array( scanPrefixKey(id) -> ScanSerializer.toBytes(app), lastUsedScanIdKey -> Shorts.toByteArray(id) )).map(_ => app) @@ -151,7 +151,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco * @param id scan identifier */ def removeScan(id: Short): Try[Unit] = - store.remove(Seq(scanPrefixKey(id))) + store.remove(Array(scanPrefixKey(id))) /** * Get scan by its identifier diff --git a/src/main/scala/org/ergoplatform/settings/CacheSettings.scala b/src/main/scala/org/ergoplatform/settings/CacheSettings.scala index ddbef605e4..ad00e343b2 100644 --- a/src/main/scala/org/ergoplatform/settings/CacheSettings.scala +++ b/src/main/scala/org/ergoplatform/settings/CacheSettings.scala @@ -13,7 +13,7 @@ case class CacheSettings( mempool: MempoolCacheSettings ) -case class HistoryCacheSettings(blockSectionsCacheSize: Int, headersCacheSize: Int, indexesCacheSize: Int) +case class HistoryCacheSettings(blockSectionsCacheSize: Int, extraCacheSize: Int, headersCacheSize: Int, indexesCacheSize: Int) case class NetworkCacheSettings( invalidModifiersCacheSize: Int, diff --git a/src/main/scala/org/ergoplatform/settings/Constants.scala b/src/main/scala/org/ergoplatform/settings/Constants.scala index b5cece06e5..c258107856 100644 --- a/src/main/scala/org/ergoplatform/settings/Constants.scala +++ b/src/main/scala/org/ergoplatform/settings/Constants.scala @@ -1,14 +1,14 @@ package org.ergoplatform.settings import org.ergoplatform.mining.difficulty.RequiredDifficulty +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionSerializer} import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} -import org.ergoplatform.modifiers.mempool.ErgoTransactionSerializer +import org.ergoplatform.modifiers.mempool.{ErgoTransaction, ErgoTransactionSerializer} import org.ergoplatform.nodeView.history.ErgoHistory.Difficulty import scorex.core.serialization.ScorexSerializer -import scorex.core.transaction.Transaction -import scorex.core.{ModifierTypeId, NodeViewModifier} +import scorex.core.NodeViewModifier import sigmastate.Values import sigmastate.Values.ErgoTree @@ -44,12 +44,12 @@ object Constants { // Number of last block headers available is scripts from ErgoStateContext val LastHeadersInContext = 10 - val modifierSerializers: Map[ModifierTypeId, ScorexSerializer[_ <: NodeViewModifier]] = + val modifierSerializers: Map[NetworkObjectTypeId.Value, ScorexSerializer[_ <: NodeViewModifier]] = Map(Header.modifierTypeId -> HeaderSerializer, Extension.modifierTypeId -> ExtensionSerializer, BlockTransactions.modifierTypeId -> BlockTransactionsSerializer, ADProofs.modifierTypeId -> ADProofsSerializer, - Transaction.ModifierTypeId -> ErgoTransactionSerializer) + ErgoTransaction.modifierTypeId -> ErgoTransactionSerializer) val SoftForkEpochs = 32 //about 45.5 days diff --git a/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala b/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala index 0aa692d7d8..0234eec6b4 100644 --- a/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala +++ b/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala @@ -47,6 +47,7 @@ case class NodeConfigurationSettings(stateType: StateType, minimalFeeAmount: Long, headerChainDiff: Int, adProofsSuffixLength: Int, + extraIndex: Boolean, blacklistedTransactions: Seq[String] = Seq.empty, checkpoint: Option[CheckpointSettings] = None) { /** @@ -84,6 +85,7 @@ trait NodeConfigurationReaders extends StateTypeReaders with CheckpointingSettin cfg.as[Long](s"$path.minimalFeeAmount"), cfg.as[Int](s"$path.headerChainDiff"), cfg.as[Int](s"$path.adProofsSuffixLength"), + cfg.as[Boolean](s"$path.extraIndex"), cfg.as[Seq[String]](s"$path.blacklistedTransactions"), cfg.as[Option[CheckpointSettings]](s"$path.checkpoint") ) diff --git a/src/main/scala/org/ergoplatform/settings/ValidationRules.scala b/src/main/scala/org/ergoplatform/settings/ValidationRules.scala index 2be07be4c3..49d516c65f 100644 --- a/src/main/scala/org/ergoplatform/settings/ValidationRules.scala +++ b/src/main/scala/org/ergoplatform/settings/ValidationRules.scala @@ -1,14 +1,13 @@ package org.ergoplatform.settings import org.ergoplatform.SigmaConstants.{MaxBoxSize, MaxPropositionBytes} -import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.modifiers.{ErgoFullBlock, NetworkObjectTypeId} import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.history.{ADProofs, BlockTransactions} import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.wallet.boxes.ErgoBoxAssetExtractor -import scorex.core.ModifierTypeId import scorex.core.validation.{InvalidModifier, ModifierValidator} import scorex.core.validation.ValidationResult.Invalid import scorex.util.ModifierId @@ -306,7 +305,7 @@ object ValidationRules { val fbDigestIncorrect: Short = 501 - def errorMessage(id: Short, details: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): String = { + def errorMessage(id: Short, details: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): String = { ValidationRules.rulesSpec(id) .invalidMod(InvalidModifier(details, modifierId, modifierTypeId)) .errors @@ -314,10 +313,10 @@ object ValidationRules { .message } - private def recoverable(errorMessage: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + private def recoverable(errorMessage: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = ModifierValidator.error(errorMessage, modifierId, modifierTypeId) - private def fatal(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + private def fatal(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = ModifierValidator.fatal(error, modifierId, modifierTypeId) } diff --git a/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala b/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala deleted file mode 100644 index 288150c80e..0000000000 --- a/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala +++ /dev/null @@ -1,236 +0,0 @@ -package org.ergoplatform.tools - -import org.ergoplatform.mining.difficulty.{DifficultyAdjustment, RequiredDifficulty} -import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.nodeView.history.ErgoHistory -import org.ergoplatform.settings.{Args, ErgoSettings, NetworkType} -import scorex.core.utils.NetworkTimeProvider - -import java.util.concurrent.TimeUnit -import scala.collection.mutable -import scala.concurrent.duration.FiniteDuration -import scala.util.Random - -object v2testing extends App { - implicit val ec: scala.concurrent.ExecutionContext = scala.concurrent.ExecutionContext.global - - private val ergoSettings: ErgoSettings = ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/mainnet.conf"), Some(NetworkType.MainNet))) - - val ntp = new NetworkTimeProvider(ergoSettings.scorexSettings.ntp) - - val ldc = new DifficultyAdjustment(ergoSettings.chainSettings) - - val eh = ErgoHistory.readOrGenerate(ergoSettings, ntp) - - val epochLength = ergoSettings.chainSettings.epochLength - - println("Chain settings: " + ergoSettings.chainSettings) - println("Epoch length: " + epochLength) - println("Block time: " + ergoSettings.chainSettings.blockInterval) - - println("best: " + eh.bestHeaderOpt.map(_.height)) - - val heights = ldc.previousHeadersRequiredForRecalculation(843776 + 1 + 1024, epochLength) - - println("hs: " + heights) - - val headerOpts = heights.map(eh.bestHeaderIdAtHeight).map(idOpt => idOpt.flatMap(id => eh.typedModifierById[Header](id))) - - println(headerOpts.map(_.map(_.id))) - - println("dir: " + ergoSettings.directory) - - val h1 = headerOpts.head.get.copy(height = 843776 + 1024, nBits = 122447235L, timestamp = System.currentTimeMillis() + 1000*60*1440*6) - - val headers = headerOpts.flatten.toSeq ++ Seq(h1) - - val diff1 = ldc.calculate(headers, epochLength) - println("diff1: " + diff1) - val nbits1 = RequiredDifficulty.encodeCompactBits(diff1) - - val h2 = headerOpts.head.get.copy(height = 843776 + 2048, nBits = nbits1, timestamp = System.currentTimeMillis() + 1000*60*1440*10) - - val headers2 = headerOpts.flatten.toSeq.tail ++ Seq(h1, h2) - val diff2 = ldc.calculate(headers2, epochLength) - println("diff2: " + diff2) - val nbits2 = RequiredDifficulty.encodeCompactBits(diff2) - - val h3 = headerOpts.head.get.copy(height = 843776 + 3072, nBits = nbits2, timestamp = System.currentTimeMillis() + (1000*60*1440*11).toInt) - val headers3 = headers2.tail ++ Seq(h3) - val diff3 = ldc.calculate(headers3, epochLength) - println("diff3: " + diff3) - val nbits3 = RequiredDifficulty.encodeCompactBits(diff3) - - val h4 = headerOpts.head.get.copy(height = 843776 + 4096, nBits = nbits3, timestamp = System.currentTimeMillis() + (1000*60*1440*11.25).toInt) - val headers4 = headers3.tail ++ Seq(h4) - val diff4 = ldc.calculate(headers4, epochLength) - println("diff4: " + diff4) - val nbits4 = RequiredDifficulty.encodeCompactBits(diff4) - - - val h5 = headerOpts.head.get.copy(height = 843776 + 4096 + 1024, nBits = nbits3, timestamp = System.currentTimeMillis() + (1000*60*1440*13.25).toInt) - val headers5 = headers4.tail ++ Seq(h5) - val diff5 = ldc.calculate(headers5, epochLength) - println("diff5: " + diff5) - val nbits5 = RequiredDifficulty.encodeCompactBits(diff5) - - - val h6 = headerOpts.head.get.copy(height = 843776 + 4096 + 2048, nBits = nbits3, timestamp = System.currentTimeMillis() + (1000*60*1440*16.5).toInt) - val headers6 = headers5.tail ++ Seq(h6) - val diff6 = ldc.calculate(headers6, epochLength) - println("diff6: " + diff6) - val nbits6 = RequiredDifficulty.encodeCompactBits(diff6) - -} - - - -object AltDiff extends App { - - implicit val ec: scala.concurrent.ExecutionContext = scala.concurrent.ExecutionContext.global - - private val currentSettings: ErgoSettings = - ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/mainnet.conf"), Some(NetworkType.MainNet))) - - private val altSettings: ErgoSettings = - ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/alt.conf"), Some(NetworkType.MainNet))) - - val ntp = new NetworkTimeProvider(altSettings.scorexSettings.ntp) - - val epochLength = altSettings.chainSettings.epochLength - - - println(currentSettings.chainSettings.epochLength) - println(altSettings.chainSettings.epochLength) - - val eh = ErgoHistory.readOrGenerate(altSettings, ntp) - - println("best: " + eh.bestHeaderOpt.map(_.height)) - - val ldc = new DifficultyAdjustment(altSettings.chainSettings) - - (1 to 843207).foreach{h => - if(h % 1024 == 1 && h > 1) { - val heights = ldc.previousHeadersRequiredForRecalculation(h, epochLength) - val headers = heights.map(eh.bestHeaderIdAtHeight).map(idOpt => idOpt.flatMap(id => eh.typedModifierById[Header](id))).flatten - val calcDiff = ldc.eip37Calculate(headers, epochLength) - val chainDiff = eh.bestHeaderAtHeight(h).get.requiredDifficulty - - println(s"calculated diff for $h: $calcDiff, chain diff: $chainDiff , difference: ${calcDiff*100/chainDiff-100}%") - } - } - -} - - -object AdaptiveSimulator extends App { - import io.circe.parser._ - - implicit val ec: scala.concurrent.ExecutionContext = scala.concurrent.ExecutionContext.global - - private val altSettings: ErgoSettings = - ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/alt.conf"), Some(NetworkType.MainNet))) - val ntp = new NetworkTimeProvider(altSettings.scorexSettings.ntp) - - val ldc = new DifficultyAdjustment(altSettings.chainSettings) - - val h1Json = - """ - |{ - | "extensionId" : "af4c9de8106960b47964d21e6eb2acdad7e3e168791e595f0806ebfb036ee7de", - | "difficulty" : "1199990374400", - | "votes" : "000000", - | "timestamp" : 1561978977137, - | "size" : 279, - | "stateRoot" : "18b7a08878f2a7ee4389c5a1cece1e2724abe8b8adc8916240dd1bcac069177303", - | "height" : 1, - | "nBits" : 100734821, - | "version" : 1, - | "id" : "b0244dfc267baca974a4caee06120321562784303a8a688976ae56170e4d175b", - | "adProofsRoot" : "766ab7a313cd2fb66d135b0be6662aa02dfa8e5b17342c05a04396268df0bfbb", - | "transactionsRoot" : "93fb06aa44413ff57ac878fda9377207d5db0e78833556b331b4d9727b3153ba", - | "extensionHash" : "0e5751c026e543b2e8ab2eb06099daa1d1e5df47778f7787faab45cdf12fe3a8", - | "powSolutions" : { - | "pk" : "03be7ad70c74f691345cbedba19f4844e7fc514e1188a7929f5ae261d5bb00bb66", - | "w" : "02da9385ac99014ddcffe88d2ac5f28ce817cd615f270a0a5eae58acfb9fd9f6a0", - | "n" : "000000030151dc63", - | "d" : 46909460813884299753486408728361968139945651324239558400157099627 - | }, - | "adProofsId" : "cfc4af9743534b30ef38deec118a85ce6f0a3741b79b7d294f3e089c118188dc", - | "transactionsId" : "fc13e7fd2d1ddbd10e373e232814b3c9ee1b6fbdc4e6257c288ecd9e6da92633", - | "parentId" : "0000000000000000000000000000000000000000000000000000000000000000" - |}""".stripMargin - - val h1 = Header.jsonDecoder.decodeJson(parse(h1Json).toOption.get).toOption.get - - var totalError = 0 - var maxDelay = 0 - - (1 to 100).foreach { _ => - var blockDelay = altSettings.chainSettings.blockInterval - val epochLength = altSettings.chainSettings.epochLength - var price = 100000 - - val medianChange = 3 // price going up 1% epoch on average - val variance = 10 // with +-20% - - val blocks = mutable.Map[Int, Header]() - blocks.put(0, h1.copy(height = 0)) // put genesis block - blocks.put(1, h1) // put genesis block - - val precision = BigInt("10000000000000000") - // t = d*c / p - // c = t*p/d - val c = blockDelay.toMillis * price * precision / h1.requiredDifficulty - println("c: " + c) - - - 129.to(32 * 1024 + 1, 128).foreach { h => - println("================================") - println("height: " + h) - - val newPrice = price + - Random.nextInt(price * medianChange / 100) + - (if (Random.nextBoolean()) { - Random.nextInt(price * variance / 100) - } else { - -Random.nextInt(price * variance / 100) - }) - /* - val epoch = (h - 1) / 128 - val newPrice = if(epoch%16 <8){ - price + Random.nextInt(price * variance / 100) - } else { - price - Random.nextInt(price * variance / 100) - } */ - - println("price: " + newPrice) - - val newBlockDelay = (blocks(h - 128).requiredDifficulty * c / precision / newPrice).toLong - - val blockBefore = h1.copy(height = h - 1, timestamp = blocks(h - 128).timestamp + 127 * newBlockDelay, nBits = blocks(h - 128).nBits) - blocks.put(h - 1, blockBefore) - - val heights = ldc.previousHeadersRequiredForRecalculation(h, epochLength) - val hs = heights.map(blocks.apply) - - val newDiff = ldc.eip37Calculate(hs, epochLength) - println("newDiff: " + newDiff) - val block = h1.copy(height = h, timestamp = blockBefore.timestamp + newBlockDelay, nBits = RequiredDifficulty.encodeCompactBits(newDiff)) - blocks.put(h, block) - - price = newPrice - blockDelay = FiniteDuration(newBlockDelay, TimeUnit.MILLISECONDS) - println("block delay: " + blockDelay.toSeconds + " s.") - totalError += Math.abs(altSettings.chainSettings.blockInterval.toMillis - blockDelay.toMillis).toInt - if (blockDelay.toSeconds > maxDelay) { - maxDelay = blockDelay.toSeconds.toInt - } - } - } - - println("Planned block time: " + altSettings.chainSettings.blockInterval) - println("Total error: " + totalError / 1000) - println("Max delay: " + maxDelay) - -} diff --git a/src/main/scala/org/ergoplatform/tools/ValidationRulesPrinter.scala b/src/main/scala/org/ergoplatform/tools/ValidationRulesPrinter.scala index b78453f0b1..ed668f4904 100644 --- a/src/main/scala/org/ergoplatform/tools/ValidationRulesPrinter.scala +++ b/src/main/scala/org/ergoplatform/tools/ValidationRulesPrinter.scala @@ -1,7 +1,7 @@ package org.ergoplatform.tools +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.settings.ValidationRules -import scorex.core.ModifierTypeId import scorex.core.validation.InvalidModifier import scorex.util.{ModifierId, ScorexLogging, bytesToId} @@ -14,7 +14,7 @@ object ValidationRulesPrinter extends App with ScorexLogging { printHeader() rules.toSeq.sortBy(_._1).foreach { r => - val rule = r._2.invalidMod(InvalidModifier("", emptyModifierId, ModifierTypeId @@ 0.toByte)).errors.head.message.trim + val rule = r._2.invalidMod(InvalidModifier("", emptyModifierId, NetworkObjectTypeId.fromByte(0))).errors.head.message.trim val activated = r._2.isActive val mayBeDisabled = r._2.mayBeDisabled val modifiers = r._2.affectedClasses.map(_.getSimpleName).mkString(", ") @@ -37,7 +37,7 @@ object ValidationRulesPrinter extends App with ScorexLogging { printHeader() } - if (r._2.invalidMod(InvalidModifier("", emptyModifierId, ModifierTypeId @@ 0.toByte)).isFatal) { + if (r._2.invalidMod(InvalidModifier("", emptyModifierId, NetworkObjectTypeId.fromByte(0))).isFatal) { // we only mention fatal errors here println(s" ${r._1} & $rule & ${boolToLatex(mayBeDisabled)} & ${boolToLatex(activated)} & $modifiers \\\\") diff --git a/src/main/scala/scorex/core/NodeViewModifier.scala b/src/main/scala/scorex/core/NodeViewModifier.scala index b2ce44fdb7..c11288a0f6 100644 --- a/src/main/scala/scorex/core/NodeViewModifier.scala +++ b/src/main/scala/scorex/core/NodeViewModifier.scala @@ -1,24 +1,27 @@ package scorex.core +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.modifiers.mempool.ErgoTransaction import scorex.core.serialization.BytesSerializable import scorex.core.utils.ScorexEncoding - sealed trait NodeViewModifier extends BytesSerializable with ScorexEncoding {self => - val modifierTypeId: ModifierTypeId + val modifierTypeId: NetworkObjectTypeId.Value //todo: check statically or dynamically output size def id: scorex.util.ModifierId - def encodedId: String = encoder.encodeId(id) - override def equals(obj: scala.Any): Boolean = obj match { case that: NodeViewModifier => (that.id == id) && (that.modifierTypeId == modifierTypeId) case _ => false } + /** + * @return readable representation of `id`, as `id` is a hex-encoded string now, just identity functions is used + */ + def encodedId: String = id + } trait EphemerealNodeViewModifier extends NodeViewModifier diff --git a/src/main/scala/scorex/core/app/Application.scala b/src/main/scala/scorex/core/app/Application.scala deleted file mode 100644 index a7224b5791..0000000000 --- a/src/main/scala/scorex/core/app/Application.scala +++ /dev/null @@ -1,157 +0,0 @@ -package scorex.core.app - -import akka.actor.{ActorRef, ActorSystem} -import akka.http.scaladsl.Http -import akka.http.scaladsl.server.{ExceptionHandler, RejectionHandler, Route} -import org.ergoplatform.ErgoApp -import org.ergoplatform.nodeView.history.ErgoSyncInfoMessageSpec -import org.ergoplatform.settings.ErgoSettings -import scorex.core.api.http.{ - ApiErrorHandler, - ApiRejectionHandler, - ApiRoute, - CompositeHttpService -} -import scorex.core.network._ -import scorex.core.network.message.Message.MessageCode -import scorex.core.network.message._ -import scorex.core.network.peer.PeerManagerRef -import scorex.core.settings.ScorexSettings -import scorex.core.utils.NetworkTimeProvider -import scorex.util.ScorexLogging - -import java.net.InetSocketAddress -import scala.concurrent.ExecutionContext - -trait Application extends ScorexLogging { - - //settings - val ergoSettings: ErgoSettings - implicit val scorexSettings: ScorexSettings - - //api - val apiRoutes: Seq[ApiRoute] - - implicit def exceptionHandler: ExceptionHandler = ApiErrorHandler.exceptionHandler - implicit def rejectionHandler: RejectionHandler = ApiRejectionHandler.rejectionHandler - - implicit protected lazy val actorSystem: ActorSystem = ActorSystem( - scorexSettings.network.agentName - ) - - implicit val executionContext: ExecutionContext = - actorSystem.dispatchers.lookup("scorex.executionContext") - - protected val features: Seq[PeerFeature] - protected val additionalMessageSpecs: Seq[MessageSpec[_]] - - //p2p - private val upnpGateway: Option[UPnPGateway] = - if (scorexSettings.network.upnpEnabled) UPnP.getValidGateway(scorexSettings.network) - else None - // TODO use available port on gateway instead settings.network.bindAddress.getPort - upnpGateway.foreach(_.addPort(scorexSettings.network.bindAddress.getPort)) - - private lazy val basicSpecs = { - Seq( - GetPeersSpec, - new PeersSpec(scorexSettings.network.maxPeerSpecObjects), - InvSpec, - RequestModifierSpec, - ModifiersSpec - ) - } - - val nodeViewHolderRef: ActorRef - val nodeViewSynchronizer: ActorRef - - /** API description in openapi format in YAML or JSON */ - val swaggerConfig: String - - val timeProvider = new NetworkTimeProvider(scorexSettings.ntp) - - //an address to send to peers - lazy val externalSocketAddress: Option[InetSocketAddress] = { - scorexSettings.network.declaredAddress orElse { - // TODO use available port on gateway instead settings.bindAddress.getPort - upnpGateway.map(u => - new InetSocketAddress( - u.externalAddress, - scorexSettings.network.bindAddress.getPort - ) - ) - } - } - - val scorexContext = ScorexContext( - messageSpecs = basicSpecs ++ additionalMessageSpecs, - upnpGateway = upnpGateway, - timeProvider = timeProvider, - externalNodeAddress = externalSocketAddress - ) - - val peerManagerRef = PeerManagerRef(ergoSettings, scorexContext) - - private val messageHandlers: ActorRef => Map[MessageCode, ActorRef] = - networkControllerRef => { - Map( - InvSpec.messageCode -> nodeViewSynchronizer, - RequestModifierSpec.messageCode -> nodeViewSynchronizer, - ModifiersSpec.messageCode -> nodeViewSynchronizer, - ErgoSyncInfoMessageSpec.messageCode -> nodeViewSynchronizer, - PeersSpec.messageCode -> PeerSynchronizerRef( - "PeerSynchronizer", - networkControllerRef, - peerManagerRef, - scorexSettings.network - ) - ) - } - - val networkControllerRef: ActorRef = - NetworkControllerRef( - "networkController", - ergoSettings, - peerManagerRef, - scorexContext, - messageHandlers - ) - - val peerSynchronizer: ActorRef = - PeerSynchronizerRef( - "PeerSynchronizer", - networkControllerRef, - peerManagerRef, - scorexSettings.network - ) - - lazy val combinedRoute: Route = CompositeHttpService( - actorSystem, - apiRoutes, - scorexSettings.restApi, - swaggerConfig - ).compositeRoute - - def run(): Unit = { - val applicationNameLimit: Int = 50 - require(scorexSettings.network.agentName.length <= applicationNameLimit) - - log.debug(s"Available processors: ${Runtime.getRuntime.availableProcessors}") - log.debug(s"Max memory available: ${Runtime.getRuntime.maxMemory}") - log.debug(s"RPC is allowed at ${scorexSettings.restApi.bindAddress.toString}") - - val bindAddress = scorexSettings.restApi.bindAddress - - Http() - .newServerAt(bindAddress.getAddress.getHostAddress, bindAddress.getPort) - .bindFlow(combinedRoute) - - //on unexpected shutdown - Runtime.getRuntime.addShutdownHook(new Thread() { - override def run() { - log.error("Unexpected shutdown") - ErgoApp.shutdownSystem() - } - }) - } -} diff --git a/src/main/scala/scorex/core/app/ScorexContext.scala b/src/main/scala/scorex/core/app/ScorexContext.scala index 82375253ff..6af361ab9a 100644 --- a/src/main/scala/scorex/core/app/ScorexContext.scala +++ b/src/main/scala/scorex/core/app/ScorexContext.scala @@ -4,9 +4,7 @@ import java.net.InetSocketAddress import scorex.core.network.UPnPGateway import scorex.core.network.message.MessageSpec -import scorex.core.utils.TimeProvider case class ScorexContext(messageSpecs: Seq[MessageSpec[_]], upnpGateway: Option[UPnPGateway], - timeProvider: TimeProvider, externalNodeAddress: Option[InetSocketAddress]) diff --git a/src/main/scala/scorex/core/consensus/ProgressInfo.scala b/src/main/scala/scorex/core/consensus/ProgressInfo.scala index d540d897ef..c292b3760f 100644 --- a/src/main/scala/scorex/core/consensus/ProgressInfo.scala +++ b/src/main/scala/scorex/core/consensus/ProgressInfo.scala @@ -1,7 +1,8 @@ package scorex.core.consensus +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.core.utils.ScorexEncoder -import scorex.core.{ModifierTypeId, PersistentNodeViewModifier} +import scorex.core.PersistentNodeViewModifier import scorex.util.ModifierId /** @@ -16,7 +17,7 @@ import scorex.util.ModifierId case class ProgressInfo[PM <: PersistentNodeViewModifier](branchPoint: Option[ModifierId], toRemove: Seq[PM], toApply: Seq[PM], - toDownload: Seq[(ModifierTypeId, ModifierId)]) + toDownload: Seq[(NetworkObjectTypeId.Value, ModifierId)]) (implicit encoder: ScorexEncoder) { if (toRemove.nonEmpty) diff --git a/src/main/scala/scorex/core/core.scala b/src/main/scala/scorex/core/core.scala index bb02ad8669..8609bd0eb1 100644 --- a/src/main/scala/scorex/core/core.scala +++ b/src/main/scala/scorex/core/core.scala @@ -1,5 +1,6 @@ package scorex +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.core.network.message.InvData import scorex.core.utils.ScorexEncoder import scorex.util.encode.Base16 @@ -7,23 +8,18 @@ import supertagged.TaggedType package object core { - //TODO implement ModifierTypeId as a trait - object ModifierTypeId extends TaggedType[Byte] - object VersionTag extends TaggedType[String] - type ModifierTypeId = ModifierTypeId.Type - type VersionTag = VersionTag.Type - def idsToString(ids: Seq[(ModifierTypeId, util.ModifierId)])(implicit enc: ScorexEncoder): String = { + def idsToString(ids: Seq[(NetworkObjectTypeId.Value, util.ModifierId)])(implicit enc: ScorexEncoder): String = { List(ids.headOption, ids.lastOption) .flatten .map { case (typeId, id) => s"($typeId,${enc.encodeId(id)})" } .mkString("[", "..", "]") } - def idsToString(modifierType: ModifierTypeId, ids: Seq[util.ModifierId])(implicit encoder: ScorexEncoder): String = { + def idsToString(modifierType: NetworkObjectTypeId.Value, ids: Seq[util.ModifierId])(implicit encoder: ScorexEncoder): String = { idsToString(ids.map(id => (modifierType, id))) } diff --git a/src/main/scala/scorex/core/network/DeliveryTracker.scala b/src/main/scala/scorex/core/network/DeliveryTracker.scala index 283d3ce622..43ecf16889 100644 --- a/src/main/scala/scorex/core/network/DeliveryTracker.scala +++ b/src/main/scala/scorex/core/network/DeliveryTracker.scala @@ -2,11 +2,11 @@ package scorex.core.network import akka.actor.Cancellable import io.circe.{Encoder, Json} +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.CheckDelivery import org.ergoplatform.nodeView.mempool.ExpiringApproximateCache import org.ergoplatform.settings.{ErgoSettings, NetworkCacheSettings} -import scorex.core.ModifierTypeId import scorex.core.consensus.ContainsModifiers import scorex.core.network.DeliveryTracker._ import scorex.core.network.ModifiersStatus._ @@ -45,10 +45,10 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, desiredSizeOfExpectingModifierQueue: Int) extends ScorexLogging with ScorexEncoding { // when a remote peer is asked for a modifier we add the requested data to `requested` - protected val requested: mutable.Map[ModifierTypeId, Map[ModifierId, RequestedInfo]] = mutable.Map() + protected val requested: mutable.Map[NetworkObjectTypeId.Value, Map[ModifierId, RequestedInfo]] = mutable.Map() // when our node received a modifier we put it to `received` - protected val received: mutable.Map[ModifierTypeId, Map[ModifierId, ConnectedPeer]] = mutable.Map() + protected val received: mutable.Map[NetworkObjectTypeId.Value, Map[ModifierId, ConnectedPeer]] = mutable.Map() private val desiredSizeOfExpectingHeaderQueue: Int = desiredSizeOfExpectingModifierQueue * 8 @@ -95,7 +95,9 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, * Since this class do not keep statuses for modifiers that are already in NodeViewHolder, * `modifierKeepers` are required here to check that modifier is in `Held` status */ - def status(modifierId: ModifierId, modifierTypeId: ModifierTypeId, modifierKeepers: Seq[ContainsModifiers[_]]): ModifiersStatus = + def status(modifierId: ModifierId, + modifierTypeId: NetworkObjectTypeId.Value, + modifierKeepers: Seq[ContainsModifiers[_]]): ModifiersStatus = if (received.get(modifierTypeId).exists(_.contains(modifierId))) Received else if (requested.get(modifierTypeId).exists(_.contains(modifierId))) Requested else if (invalidModifierCache.mightContain(modifierId)) Invalid @@ -112,7 +114,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, /** * Set status of modifier with id `id` to `Requested` */ - def setRequested(typeId: ModifierTypeId, + def setRequested(typeId: NetworkObjectTypeId.Value, id: ModifierId, supplier: ConnectedPeer, checksDone: Int = 0) @@ -128,12 +130,12 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, /** * @return - information on requested modifier delivery tracker potentially has */ - def getRequestedInfo(typeId: ModifierTypeId, id: ModifierId): Option[RequestedInfo] = { + def getRequestedInfo(typeId: NetworkObjectTypeId.Value, id: ModifierId): Option[RequestedInfo] = { requested.get(typeId).flatMap(_.get(id)) } /** Get peer we're communicating with in regards with modifier `id` **/ - def getSource(id: ModifierId, modifierTypeId: ModifierTypeId): Option[ConnectedPeer] = { + def getSource(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Option[ConnectedPeer] = { status(id, modifierTypeId, Seq.empty) match { case Requested => requested.get(modifierTypeId).flatMap(_.get(id)).map(_.peer) case Received => received.get(modifierTypeId).flatMap(_.get(id)) @@ -145,7 +147,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, * Modified with id `id` is permanently invalid - set its status to `Invalid` * and return [[ConnectedPeer]] which sent bad modifier. */ - def setInvalid(id: ModifierId, modifierTypeId: ModifierTypeId): Option[ConnectedPeer] = { + def setInvalid(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Option[ConnectedPeer] = { val oldStatus: ModifiersStatus = status(id, modifierTypeId, Seq.empty) val transitionCheck = tryWithLogging { checkStatusTransition(oldStatus, Invalid) @@ -188,7 +190,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, /** * Modifier with id `id` was successfully applied to history - set its status to `Held`. */ - def setHeld(id: ModifierId, modifierTypeId: ModifierTypeId): Unit = + def setHeld(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Unit = tryWithLogging { val oldStatus = status(id, modifierTypeId, Seq.empty) checkStatusTransition(oldStatus, Held) @@ -203,7 +205,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, * this modifier was removed from cache because cache is overfull or * we stop trying to download this modifiers due to exceeded number of retries */ - def setUnknown(id: ModifierId, modifierTypeId: ModifierTypeId): Unit = + def setUnknown(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Unit = tryWithLogging { val oldStatus = status(id, modifierTypeId, Seq.empty) checkStatusTransition(oldStatus, Unknown) @@ -213,7 +215,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, /** * Modifier with id `id` was received from remote peer - set its status to `Received`. */ - def setReceived(id: ModifierId, modifierTypeId: ModifierTypeId, sender: ConnectedPeer): Unit = + def setReceived(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value, sender: ConnectedPeer): Unit = tryWithLogging { val oldStatus = status(id, modifierTypeId, Seq.empty) checkStatusTransition(oldStatus, Received) @@ -251,7 +253,10 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, case _ => false } - def clearStatusForModifier(id: ModifierId, modifierTypeId: ModifierTypeId, oldStatus: ModifiersStatus): Unit = + /** + * Clear old known status `oldStatus` from tracked block section `modifierId` of type `modifierTypeId` + */ + def clearStatusForModifier(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value, oldStatus: ModifiersStatus): Unit = oldStatus match { case Requested => requested.flatAdjust(modifierTypeId)(_.map { infoById => @@ -325,16 +330,16 @@ object DeliveryTracker { } case class FullInfo( - invalidModifierApproxSize: Long, - requested: Seq[(ModifierTypeId, Map[ModifierId, RequestedInfo])], - received: Seq[(ModifierTypeId, Map[ModifierId, ConnectedPeer])] + invalidModifierApproxSize: Long, + requested: Seq[(NetworkObjectTypeId.Value, Map[ModifierId, RequestedInfo])], + received: Seq[(NetworkObjectTypeId.Value, Map[ModifierId, ConnectedPeer])] ) object FullInfo { import io.circe.syntax._ implicit val encodeState: Encoder[FullInfo] = new Encoder[FullInfo] { - def nestedMapAsJson[T : Encoder](requested: Seq[(ModifierTypeId, Map[ModifierId, T])]): Json = + def nestedMapAsJson[T : Encoder](requested: Seq[(NetworkObjectTypeId.Value, Map[ModifierId, T])]): Json = Json.obj( requested.map { case (k, v) => k.toString -> Json.obj(v.mapValues(_.asJson).toSeq:_*) diff --git a/src/main/scala/scorex/core/network/NetworkController.scala b/src/main/scala/scorex/core/network/NetworkController.scala index 9ccdfd5c07..6890109704 100644 --- a/src/main/scala/scorex/core/network/NetworkController.scala +++ b/src/main/scala/scorex/core/network/NetworkController.scala @@ -9,13 +9,14 @@ import akka.util.Timeout import scorex.core.app.{ScorexContext, Version} import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.{DisconnectedPeer, HandshakedPeer} import org.ergoplatform.network.ModePeerFeature +import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.settings.ErgoSettings import scorex.core.network.message.Message.MessageCode import scorex.core.network.message.Message import scorex.core.network.peer.PeerManager.ReceivableMessages._ import scorex.core.network.peer.{LocalAddressPeerFeature, PeerInfo, PeerManager, PeersStatus, PenaltyType, RestApiUrlPeerFeature, SessionIdPeerFeature} -import scorex.core.utils.TimeProvider.Time -import scorex.core.utils.{NetworkUtils, TimeProvider} +import org.ergoplatform.nodeView.history.ErgoHistory.Time +import scorex.core.utils.NetworkUtils import scorex.util.ScorexLogging import scala.concurrent.ExecutionContext @@ -68,7 +69,7 @@ class NetworkController(ergoSettings: ErgoSettings, * Storing timestamp of a last message got via p2p network. * Used to check whether connectivity is lost. */ - private var lastIncomingMessageTime: TimeProvider.Time = 0L + private var lastIncomingMessageTime: ErgoHistory.Time = 0L //check own declared address for validity validateDeclaredAddress() @@ -111,7 +112,7 @@ class NetworkController(ergoSettings: ErgoSettings, context stop self } - private def networkTime(): Time = scorexContext.timeProvider.time() + private def time(): Time = System.currentTimeMillis() private def businessLogic: Receive = { //a message coming in from another peer @@ -125,7 +126,7 @@ class NetworkController(ergoSettings: ErgoSettings, val remoteAddress = remote.connectionId.remoteAddress connections.get(remoteAddress) match { case Some(cp) => - val now = networkTime() + val now = time() lastIncomingMessageTime = now cp.lastMessage = now case None => log.warn("Connection not found for a message got from: " + remoteAddress) @@ -180,7 +181,7 @@ class NetworkController(ergoSettings: ErgoSettings, handlerRef ! Close case Handshaked(connectedPeer) => - val now = networkTime() + val now = time() lastIncomingMessageTime = now handleHandshake(connectedPeer, sender()) @@ -194,7 +195,7 @@ class NetworkController(ergoSettings: ErgoSettings, // If a message received from p2p within connection timeout, // connectivity is not lost thus we're removing the peer // we add multiplier 6 to remove more dead peers (and still not dropping a lot when connectivity lost) - val noNetworkMessagesFor = networkTime() - lastIncomingMessageTime + val noNetworkMessagesFor = time() - lastIncomingMessageTime if (noNetworkMessagesFor < networkSettings.connectionTimeout.toMillis * 6) { peerManagerRef ! RemovePeer(c.remoteAddress) } @@ -218,7 +219,7 @@ class NetworkController(ergoSettings: ErgoSettings, //calls from API / application private def interfaceCalls: Receive = { case GetPeersStatus => - sender() ! PeersStatus(lastIncomingMessageTime, networkTime()) + sender() ! PeersStatus(lastIncomingMessageTime, time()) case GetConnectedPeers => sender() ! connections.values.filter(_.peerInfo.nonEmpty) @@ -286,7 +287,7 @@ class NetworkController(ergoSettings: ErgoSettings, context.system.scheduler.scheduleWithFixedDelay(60.seconds, 60.seconds) { () => { // Drop connections with peers if they seem to be inactive - val now = networkTime() + val now = time() connections.values.foreach { cp => val lastSeen = cp.lastMessage val timeout = networkSettings.inactiveConnectionDeadline.toMillis @@ -373,7 +374,7 @@ class NetworkController(ergoSettings: ErgoSettings, val handler = context.actorOf(handlerProps) // launch connection handler context.watch(handler) - val connectedPeer = ConnectedPeer(connectionId, handler, networkTime(), None) + val connectedPeer = ConnectedPeer(connectionId, handler, time(), None) connections += connectionId.remoteAddress -> connectedPeer unconfirmedConnections -= connectionId.remoteAddress } diff --git a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala index 6ef80138fb..ad3f850a9e 100644 --- a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala +++ b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala @@ -72,7 +72,7 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, val peerInfo = PeerInfo( receivedHandshake.peerSpec, - scorexContext.timeProvider.time(), + System.currentTimeMillis(), Some(direction) ) val peer = ConnectedPeer(connectionDescription.connectionId, self, 0, Some(peerInfo)) @@ -243,7 +243,7 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, ownSocketAddress, localFeatures ), - scorexContext.timeProvider.time() + System.currentTimeMillis() ) } diff --git a/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala b/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala index 7e09f33756..396a63c676 100644 --- a/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala +++ b/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala @@ -1,20 +1,24 @@ package scorex.core.network.message +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.core.consensus.SyncInfo import scorex.core.network._ import scorex.core.network.message.Message.MessageCode import scorex.core.serialization.ScorexSerializer -import scorex.core.{ModifierTypeId, NodeViewModifier} +import scorex.core.NodeViewModifier import scorex.util.Extensions._ import scorex.util.serialization.{Reader, Writer} import scorex.util.{ModifierId, ScorexLogging, bytesToId, idToBytes} import scala.collection.immutable -case class ModifiersData(typeId: ModifierTypeId, modifiers: Map[ModifierId, Array[Byte]]) +/** + * Wrapper for block sections of the same type. Used to send multiple block sections at once ove the wire. + */ +case class ModifiersData(typeId: NetworkObjectTypeId.Value, modifiers: Map[ModifierId, Array[Byte]]) -case class InvData(typeId: ModifierTypeId, ids: Seq[ModifierId]) +case class InvData(typeId: NetworkObjectTypeId.Value, ids: Seq[ModifierId]) /** * The `SyncInfo` message requests an `Inv` message that provides modifier ids @@ -63,7 +67,7 @@ object InvSpec extends MessageSpecV1[InvData] { } override def parse(r: Reader): InvData = { - val typeId = ModifierTypeId @@ r.getByte() + val typeId = NetworkObjectTypeId.fromByte(r.getByte()) val count = r.getUInt().toIntExact require(count > 0, "empty inv list") require(count <= maxInvObjects, s"$count elements in a message while limit is $maxInvObjects") @@ -142,7 +146,7 @@ object ModifiersSpec extends MessageSpecV1[ModifiersData] with ScorexLogging { } override def parse(r: Reader): ModifiersData = { - val typeId = ModifierTypeId @@ r.getByte() // 1 byte + val typeId = NetworkObjectTypeId.fromByte(r.getByte()) // 1 byte val count = r.getUInt().toIntExact // 8 bytes require(count > 0, s"Illegal message with 0 modifiers of type $typeId") val resMap = immutable.Map.newBuilder[ModifierId, Array[Byte]] diff --git a/src/main/scala/scorex/core/network/peer/PeerDatabase.scala b/src/main/scala/scorex/core/network/peer/PeerDatabase.scala index 841b9790fc..086ccba786 100644 --- a/src/main/scala/scorex/core/network/peer/PeerDatabase.scala +++ b/src/main/scala/scorex/core/network/peer/PeerDatabase.scala @@ -1,9 +1,10 @@ package scorex.core.network.peer +import org.ergoplatform.nodeView.history.ErgoHistory + import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream} import java.net.{InetAddress, InetSocketAddress} import org.ergoplatform.settings.ErgoSettings -import scorex.core.utils.TimeProvider import scorex.db.LDBFactory import scorex.util.ScorexLogging @@ -13,7 +14,7 @@ import scala.util.{Failure, Success, Try} /** * In-memory peer database implementation supporting temporal blacklisting. */ -final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) extends ScorexLogging { +final class PeerDatabase(settings: ErgoSettings) extends ScorexLogging { private val objectStore = LDBFactory.createKvDb(s"${settings.directory}/peers") @@ -29,7 +30,7 @@ final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) ext /** * banned peer ip -> ban expiration timestamp */ - private var blacklist = Map.empty[InetAddress, TimeProvider.Time] + private var blacklist = Map.empty[InetAddress, ErgoHistory.Time] /** * penalized peer ip -> (accumulated penalty score, last penalty timestamp) @@ -85,7 +86,7 @@ final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) ext Option(socketAddress.getAddress).foreach { address => penaltyBook -= address if (!blacklist.keySet.contains(address)){ - blacklist += address -> (timeProvider.time() + penaltyDuration(penaltyType)) + blacklist += address -> (System.currentTimeMillis() + penaltyDuration(penaltyType)) } else { log.warn(s"${address.toString} is already blacklisted") } @@ -126,7 +127,7 @@ final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) ext */ def penalize(socketAddress: InetSocketAddress, penaltyType: PenaltyType): Boolean = Option(socketAddress.getAddress).exists { address => - val currentTime = timeProvider.time() + val currentTime = System.currentTimeMillis() val safeInterval = settings.scorexSettings.network.penaltySafeInterval.toMillis val (penaltyScoreAcc, lastPenaltyTs) = penaltyBook.getOrElse(address, (0, 0L)) val applyPenalty = currentTime - lastPenaltyTs - safeInterval > 0 || penaltyType.isPermanent @@ -139,7 +140,7 @@ final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) ext if (newPenaltyScore > settings.scorexSettings.network.penaltyScoreThreshold) { true } else { - penaltyBook += address -> (newPenaltyScore -> timeProvider.time()) + penaltyBook += address -> (newPenaltyScore -> System.currentTimeMillis()) false } } @@ -154,7 +155,7 @@ final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) ext Option(socketAddress.getAddress).map(penaltyScore).getOrElse(0) private def checkBanned(address: InetAddress, bannedTill: Long): Boolean = { - val stillBanned = timeProvider.time() < bannedTill + val stillBanned = System.currentTimeMillis() < bannedTill if (!stillBanned) removeFromBlacklist(address) stillBanned } diff --git a/src/main/scala/scorex/core/network/peer/PeerManager.scala b/src/main/scala/scorex/core/network/peer/PeerManager.scala index 94909682d3..82cc399493 100644 --- a/src/main/scala/scorex/core/network/peer/PeerManager.scala +++ b/src/main/scala/scorex/core/network/peer/PeerManager.scala @@ -19,7 +19,7 @@ class PeerManager(settings: ErgoSettings, scorexContext: ScorexContext) extends import PeerManager.ReceivableMessages._ - private val peerDatabase = new PeerDatabase(settings, scorexContext.timeProvider) + private val peerDatabase = new PeerDatabase(settings) if (peerDatabase.isEmpty) { // fill database with peers from config file if empty diff --git a/src/main/scala/scorex/core/settings/Settings.scala b/src/main/scala/scorex/core/settings/Settings.scala index ab96e5f675..c67890f61b 100644 --- a/src/main/scala/scorex/core/settings/Settings.scala +++ b/src/main/scala/scorex/core/settings/Settings.scala @@ -6,7 +6,6 @@ import com.typesafe.config.{Config, ConfigFactory} import net.ceedubs.ficus.Ficus._ import net.ceedubs.ficus.readers.ArbitraryTypeReader._ import scorex.core.network.message.Message -import scorex.core.utils.NetworkTimeProviderSettings import scorex.util.ScorexLogging import scala.concurrent.duration._ @@ -57,8 +56,7 @@ case class ScorexSettings(dataDir: File, logDir: File, logging: LoggingSettings, network: NetworkSettings, - restApi: RESTApiSettings, - ntp: NetworkTimeProviderSettings) + restApi: RESTApiSettings) object ScorexSettings extends ScorexLogging with SettingsReaders { diff --git a/src/main/scala/scorex/core/transaction/Transaction.scala b/src/main/scala/scorex/core/transaction/Transaction.scala index ddda38caa4..c170967ad1 100644 --- a/src/main/scala/scorex/core/transaction/Transaction.scala +++ b/src/main/scala/scorex/core/transaction/Transaction.scala @@ -1,6 +1,7 @@ package scorex.core.transaction -import scorex.core.{EphemerealNodeViewModifier, ModifierTypeId} +import org.ergoplatform.modifiers.{NetworkObjectTypeId, TransactionTypeId} +import scorex.core.EphemerealNodeViewModifier import scorex.crypto.hash.Blake2b256 import scorex.util.{ModifierId, bytesToId} @@ -9,14 +10,9 @@ import scorex.util.{ModifierId, bytesToId} * A transaction is an atomic state modifier */ trait Transaction extends EphemerealNodeViewModifier { - override val modifierTypeId: ModifierTypeId = Transaction.ModifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = TransactionTypeId.value val messageToSign: Array[Byte] override lazy val id: ModifierId = bytesToId(Blake2b256(messageToSign)) } - - -object Transaction { - val ModifierTypeId: scorex.core.ModifierTypeId = scorex.core.ModifierTypeId @@ 2.toByte -} diff --git a/src/main/scala/scorex/core/utils/NetworkTime.scala b/src/main/scala/scorex/core/utils/NetworkTime.scala deleted file mode 100644 index 528aa38b05..0000000000 --- a/src/main/scala/scorex/core/utils/NetworkTime.scala +++ /dev/null @@ -1,59 +0,0 @@ -package scorex.core.utils - -import java.net.InetAddress -import java.util.concurrent.atomic.AtomicLong - -import org.apache.commons.net.ntp.NTPUDPClient - -import scala.concurrent.duration._ -import scala.concurrent.{ExecutionContext, Future} -import scala.util.{Failure, Success} - -object NetworkTime { - def localWithOffset(offset: Long): Long = System.currentTimeMillis() + offset - type Offset = Long -} - -case class NetworkTimeProviderSettings(server: String, updateEvery: FiniteDuration, timeout: FiniteDuration) - -class NetworkTimeProvider(ntpSettings: NetworkTimeProviderSettings)(implicit ec: ExecutionContext) - extends TimeProvider with scorex.util.ScorexLogging { - - private val lastUpdate = new AtomicLong(0) - private val offset = new AtomicLong(0) - private val client = new NTPUDPClient() - client.setDefaultTimeout(ntpSettings.timeout.toMillis.toInt) - client.open() - - override def time(): TimeProvider.Time = { - checkUpdateRequired() - NetworkTime.localWithOffset(offset.get()) - } - - private def updateOffset(): Future[NetworkTime.Offset] = Future { - val info = client.getTime(InetAddress.getByName(ntpSettings.server)) - info.computeDetails() - info.getOffset - } - - private def checkUpdateRequired(): Unit = { - val time = NetworkTime.localWithOffset(offset.get()) - // set lastUpdate to current time so other threads won't start to update it - val lu = lastUpdate.getAndSet(time) - if (time > lu + ntpSettings.updateEvery.toMillis) { - // time to update offset - updateOffset().onComplete { - case Success(newOffset) => - offset.set(newOffset) - log.info("New offset adjusted: " + offset) - lastUpdate.set(time) - case Failure(e) => - log.warn("Problems with NTP: ", e) - lastUpdate.compareAndSet(time, lu) - } - } else { - // No update required. Set lastUpdate back to it's initial value - lastUpdate.compareAndSet(time, lu) - } - } -} diff --git a/src/main/scala/scorex/core/utils/TimeProvider.scala b/src/main/scala/scorex/core/utils/TimeProvider.scala deleted file mode 100644 index a2ab1e9b99..0000000000 --- a/src/main/scala/scorex/core/utils/TimeProvider.scala +++ /dev/null @@ -1,9 +0,0 @@ -package scorex.core.utils - -object TimeProvider { - type Time = Long -} - -trait TimeProvider { - def time(): TimeProvider.Time -} diff --git a/src/main/scala/scorex/core/validation/ModifierError.scala b/src/main/scala/scorex/core/validation/ModifierError.scala index d8fce64b30..a80444668a 100644 --- a/src/main/scala/scorex/core/validation/ModifierError.scala +++ b/src/main/scala/scorex/core/validation/ModifierError.scala @@ -1,11 +1,15 @@ package scorex.core.validation -import scorex.core.ModifierTypeId +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.util.ModifierId import scala.util.control.NoStackTrace -case class InvalidModifier(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId) +/** + * Container for error details in regards with block section turned out to be invalid. Wraps validation error, + * block section id, and block section type id. + */ +case class InvalidModifier(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value) /** Base trait for errors that were occurred during NodeView Modifier validation */ @@ -13,7 +17,7 @@ trait ModifierError { def message: String def isFatal: Boolean def modifierId: ModifierId - def modifierTypeId: ModifierTypeId + def modifierTypeId: NetworkObjectTypeId.Value def toThrowable: Throwable def info: String = { @@ -25,7 +29,7 @@ trait ModifierError { /** Permanent modifier error that could not be recovered in future even after any history updates */ @SuppressWarnings(Array("org.wartremover.warts.Null")) -class MalformedModifierError(val message: String, val modifierId: ModifierId, val modifierTypeId: ModifierTypeId, cause: Option[Throwable] = None) +class MalformedModifierError(val message: String, val modifierId: ModifierId, val modifierTypeId: NetworkObjectTypeId.Value, cause: Option[Throwable] = None) extends Exception(message, cause.orNull) with ModifierError { def isFatal: Boolean = true def toThrowable: Throwable = this @@ -35,7 +39,7 @@ class MalformedModifierError(val message: String, val modifierId: ModifierId, va * When an instance is created, the stack trace is not collected which makes this exception lightweight. */ @SuppressWarnings(Array("org.wartremover.warts.Null")) -class RecoverableModifierError(val message: String, val modifierId: ModifierId, val modifierTypeId: ModifierTypeId, cause: Option[Throwable] = None) +class RecoverableModifierError(val message: String, val modifierId: ModifierId, val modifierTypeId: NetworkObjectTypeId.Value, cause: Option[Throwable] = None) extends Exception(message, cause.orNull) with ModifierError with NoStackTrace { def isFatal: Boolean = false def toThrowable: Throwable = this diff --git a/src/main/scala/scorex/core/validation/ModifierValidator.scala b/src/main/scala/scorex/core/validation/ModifierValidator.scala index ed5b4e7ec0..8e23763c69 100644 --- a/src/main/scala/scorex/core/validation/ModifierValidator.scala +++ b/src/main/scala/scorex/core/validation/ModifierValidator.scala @@ -1,7 +1,6 @@ package scorex.core.validation - -import scorex.core.ModifierTypeId +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.core.consensus.ModifierSemanticValidity import scorex.core.utils.ScorexEncoder import scorex.core.validation.ValidationResult._ @@ -29,33 +28,31 @@ object ModifierValidator { } /** report recoverable modifier error that could be fixed by later retries */ - def error(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + def error(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = invalid(new RecoverableModifierError(error, modifierId, modifierTypeId, None)) /** report recoverable modifier error that could be fixed by later retries */ - def error(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId, cause: Throwable): Invalid = + def error(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value, cause: Throwable): Invalid = invalid(new RecoverableModifierError(msg(error, cause), modifierId, modifierTypeId, Option(cause))) /** report recoverable modifier error that could be fixed by later retries */ - def error(description: String, detail: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + def error(description: String, detail: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = error(msg(description, detail), modifierId, modifierTypeId) /** report non-recoverable modifier error that could not be fixed by retries and requires modifier change */ - def fatal(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + def fatal(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = invalid(new MalformedModifierError(error, modifierId, modifierTypeId, None)) /** report non-recoverable modifier error that could not be fixed by retries and requires modifier change */ - def fatal(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId, cause: Throwable): Invalid = + def fatal(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value, cause: Throwable): Invalid = invalid(new MalformedModifierError(msg(error, cause), modifierId, modifierTypeId, Option(cause))) /** report non-recoverable modifier error that could not be fixed by retries and requires modifier change */ - def fatal(description: String, detail: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + def fatal(description: String, detail: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = fatal(msg(description, detail), modifierId, modifierTypeId) /** unsuccessful validation with a given error; also logs the error as an exception */ - def invalid(error: ModifierError): Invalid = { - Invalid(Seq(error)) - } + def invalid(error: ModifierError): Invalid = Invalid(Seq(error)) /** successful validation without payload */ val success: Valid[Unit] = Valid(()) @@ -103,7 +100,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate the first argument equals the second. This should not be used with `ModifierId` of type `Array[Byte]`. * The `error` callback will be provided with detail on argument values for better reporting */ - def validateEquals[A](id: Short, given: => A, expected: => A, modifierId: ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateEquals[A](id: Short, given: => A, expected: => A, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass((given, expected) match { case _ if !settings.isActive(id) => result case (a: Array[_], b: Array[_]) if a sameElements[Any] b => result @@ -115,7 +112,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate the `id`s are equal. The `error` callback will be provided with detail on argument values */ - def validateEqualIds(id: Short, given: => ModifierId, expected: => ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateEqualIds(id: Short, given: => ModifierId, expected: => ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass { if (!settings.isActive(id) || given == expected) result else settings.getError(id, InvalidModifier(s"Given: ${e.encodeId(given)}, expected ${e.encodeId(expected)}", given, modifierTypeId)) @@ -131,7 +128,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate the `condition` is `Success`. Otherwise the `error` callback will be provided with detail * on a failure exception */ - def validateNoFailure(id: Short, condition: => Try[_], modifierId: ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateNoFailure(id: Short, condition: => Try[_], modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass(if (!settings.isActive(id)) result else condition.fold(e => settings.getError(id, e, modifierId, modifierTypeId), _ => result)) } @@ -143,7 +140,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS * @param modifierTypeId provide for a case when it cannot be resolved from a ModifierError * @return validation state */ - def validateNoFailure(id: Short, condition: => Try[_], modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateNoFailure(id: Short, condition: => Try[_], modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass { if (!settings.isActive(id)) { result @@ -163,7 +160,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate the `block` doesn't throw an Exception. Otherwise the `error` callback will be provided with detail * on the exception */ - def validateNoThrow(id: Short, block: => Any, modifierId: ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateNoThrow(id: Short, block: => Any, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { validateNoFailure(id, Try(block), modifierId, modifierTypeId) } @@ -176,7 +173,8 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate `condition` against payload is `true` or else return the `error` */ - def validateTryFlatten(id: Short, operation: T => Try[T], condition: T => Boolean, modifierId: ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateTryFlatten(id: Short, operation: T => Try[T], condition: T => Boolean, + modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass(result.toTry.flatMap(r => operation(r)) match { case Failure(ex) => settings.getError(id, ex, modifierId, modifierTypeId) case Success(v) if settings.isActive(id) && !condition(v) => settings.getError(id, InvalidModifier(modifierId, modifierId, modifierTypeId)) @@ -200,7 +198,8 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS * If given option is `None` then pass the previous result as success. * Return `error` if option is `Some` amd condition is `false` */ - def validateOrSkipFlatten[A](id: Short, option: => Option[A], condition: A => Boolean, modifierId: ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateOrSkipFlatten[A](id: Short, option: => Option[A], condition: A => Boolean, + modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass(option match { case Some(v) if settings.isActive(id) && !condition(v) => settings.getError(id, InvalidModifier(modifierId, modifierId, modifierTypeId)) case _ => result diff --git a/src/main/scala/scorex/core/validation/ValidationSettings.scala b/src/main/scala/scorex/core/validation/ValidationSettings.scala index 657b4a005e..cabd24da9d 100644 --- a/src/main/scala/scorex/core/validation/ValidationSettings.scala +++ b/src/main/scala/scorex/core/validation/ValidationSettings.scala @@ -1,6 +1,6 @@ package scorex.core.validation -import scorex.core.ModifierTypeId +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.core.validation.ValidationResult.Invalid import scorex.util.ModifierId @@ -11,7 +11,10 @@ import scorex.util.ModifierId abstract class ValidationSettings { val isFailFast: Boolean - def getError(id: Short, e: Throwable, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + /** + * @return validation error of type `id` for block section `modifierId` of type `modifierTypeId`, error details in `e` + */ + def getError(id: Short, e: Throwable, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = getError(id, InvalidModifier(e.getMessage, modifierId, modifierTypeId)) def getError(id: Short, invalidMod: InvalidModifier): ValidationResult.Invalid diff --git a/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala b/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala index b230d6d7c0..c201ebd15d 100644 --- a/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala +++ b/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala @@ -11,10 +11,10 @@ class KvStoreReaderSpec extends AnyPropSpec with Matchers with DBSpec { val keyEnd = byteString("Z") store.getRange(keyStart, keyEnd).length shouldBe 0 - store.insert(Seq(keyStart -> keyStart)).get + store.insert(Array(keyStart -> keyStart)).get store.getRange(keyStart, keyEnd).length shouldBe 1 - store.insert(Seq(keyEnd -> keyEnd)).get + store.insert(Array(keyEnd -> keyEnd)).get store.getRange(keyStart, keyEnd).length shouldBe 2 // keys before the range @@ -27,7 +27,7 @@ class KvStoreReaderSpec extends AnyPropSpec with Matchers with DBSpec { store.getRange(byteString("<"), byteString("?")).length shouldBe 0 //removing keys - store.remove(Seq(keyStart, keyEnd)).get + store.remove(Array(keyStart, keyEnd)).get store.getRange(keyStart, keyEnd).length shouldBe 0 } } diff --git a/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala b/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala index 0271607ec3..cf675c916d 100644 --- a/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala +++ b/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala @@ -10,14 +10,14 @@ class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { val valueA = (byteString("A"), byteString("1")) val valueB = (byteString("B"), byteString("2")) - store.update(toInsert = Seq(valueA, valueB), toRemove = Seq.empty).get + store.update(toInsert = Array(valueA, valueB), toRemove = Array.empty).get store.get(valueA._1).toBs shouldBe Some(valueA._2).toBs store.get(valueB._1).toBs shouldBe Some(valueB._2).toBs store.getAll.toSeq.toBs shouldBe Seq(valueA, valueB).toBs - store.update(toInsert = Seq.empty, toRemove = Seq(valueA._1)).get + store.update(toInsert = Array.empty, toRemove = Array(valueA._1)).get store.get(valueA._1) shouldBe None } } @@ -28,11 +28,11 @@ class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { val valA = byteString("1") val valB = byteString("2") - store.insert(Seq(key -> valA)).get + store.insert(Array(key -> valA)).get store.get(key).toBs shouldBe Some(valA).toBs - store.insert(Seq(key -> valB)).get + store.insert(Array(key -> valB)).get store.get(key).toBs shouldBe Some(valB).toBs @@ -49,7 +49,7 @@ class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { val valueE = (byteString("E"), byteString("3")) val valueF = (byteString("F"), byteString("4")) - store.insert(Seq(valueA, valueB, valueC, valueD, valueE, valueF)).get + store.insert(Array(valueA, valueB, valueC, valueD, valueE, valueF)).get store.lastKeyInRange(valueA._1, valueC._1).get.toSeq shouldBe valueC._1.toSeq store.lastKeyInRange(valueD._1, valueF._1).get.toSeq shouldBe valueF._1.toSeq diff --git a/src/test/scala/org/ergoplatform/http/routes/ErgoPeersApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ErgoPeersApiRouteSpec.scala index d28a8339a0..de1a6edc5d 100644 --- a/src/test/scala/org/ergoplatform/http/routes/ErgoPeersApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/ErgoPeersApiRouteSpec.scala @@ -16,8 +16,6 @@ import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks import scorex.core.network.NetworkController.ReceivableMessages.GetConnectedPeers import scorex.core.network.peer.PeerManager.ReceivableMessages.GetAllPeers import scorex.core.settings.RESTApiSettings -import scorex.core.utils.NetworkTimeProvider -import scorex.core.utils.TimeProvider.Time import java.net.InetSocketAddress import scala.concurrent.Future @@ -30,10 +28,6 @@ class ErgoPeersApiRouteSpec extends AnyFlatSpec with ScalaCheckPropertyChecks with Stubs { - val fakeTimeProvider: NetworkTimeProvider = new NetworkTimeProvider(settings.scorexSettings.ntp) { - override def time(): Time = 123 - } - implicit val actorTimeout: Timeout = Timeout(15.seconds.dilated) implicit val routeTimeout: RouteTestTimeout = RouteTestTimeout(15.seconds.dilated) diff --git a/src/test/scala/org/ergoplatform/http/routes/InfoApiRoutesSpec.scala b/src/test/scala/org/ergoplatform/http/routes/InfoApiRoutesSpec.scala index a599cb4ca6..ebe6f17110 100644 --- a/src/test/scala/org/ergoplatform/http/routes/InfoApiRoutesSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/InfoApiRoutesSpec.scala @@ -21,8 +21,6 @@ import org.ergoplatform.nodeView.history.ErgoHistory.Difficulty import org.ergoplatform.utils.Stubs import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import scorex.core.utils.TimeProvider.Time -import scorex.core.utils.NetworkTimeProvider import scala.concurrent.duration._ import scala.concurrent.{Await, Future} @@ -33,14 +31,10 @@ class InfoApiRoutesSpec extends AnyFlatSpec with FailFastCirceSupport with Stubs { - val fakeTimeProvider: NetworkTimeProvider = new NetworkTimeProvider(settings.scorexSettings.ntp) { - override def time(): Time = 123 - } - implicit val actorTimeout: Timeout = Timeout(15.seconds.dilated) implicit val routeTimeout: RouteTestTimeout = RouteTestTimeout(15.seconds.dilated) - val statsCollector: ActorRef = ErgoStatsCollectorRef(nodeViewRef, networkControllerRef, null, settings, fakeTimeProvider) - val route: Route = InfoApiRoute(statsCollector, settings.scorexSettings.restApi, fakeTimeProvider).route + val statsCollector: ActorRef = ErgoStatsCollectorRef(nodeViewRef, networkControllerRef, null, settings) + val route: Route = InfoApiRoute(statsCollector, settings.scorexSettings.restApi).route val requiredDifficulty = BigInt(1) override def beforeAll: Unit = { @@ -57,7 +51,7 @@ class InfoApiRoutesSpec extends AnyFlatSpec c.downField("appVersion").as[String] shouldEqual Right(Version.VersionString) c.downField("stateType").as[String] shouldEqual Right(settings.nodeSettings.stateType.stateTypeName) c.downField("isMining").as[Boolean] shouldEqual Right(settings.nodeSettings.mining) - c.downField("launchTime").as[Long] shouldEqual Right(fakeTimeProvider.time()) + (System.currentTimeMillis() - c.downField("launchTime").as[Long].toOption.getOrElse(0L)) < 2000 shouldBe true c.downField("eip27Supported").as[Boolean] shouldEqual Right(true) c.downField("restApiUrl").as[String] shouldEqual Right("https://example.com:80") } diff --git a/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala index a865e9708f..f261e1769a 100644 --- a/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala @@ -67,7 +67,7 @@ class ScriptApiRouteSpec extends AnyFlatSpec val assertion = (json: Json) => { status shouldBe StatusCodes.OK val addressStr = json.hcursor.downField("address").as[String].right.get - ergoAddressEncoder.fromString(addressStr).get.addressTypePrefix shouldEqual Pay2SAddress.addressTypePrefix + addressEncoder.fromString(addressStr).get.addressTypePrefix shouldEqual Pay2SAddress.addressTypePrefix } Post(prefix + suffix, Json.obj("source" -> scriptSource.asJson)) ~> route ~> check(assertion(responseAs[Json])) Post(prefix + suffix, Json.obj("source" -> scriptSourceSigProp.asJson)) ~> route ~> @@ -97,9 +97,9 @@ class ScriptApiRouteSpec extends AnyFlatSpec val tree = ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(Base16.decode(treeStr).get) - val addr = ergoAddressEncoder.fromProposition(tree).get + val addr = addressEncoder.fromProposition(tree).get - ergoAddressEncoder.toString(addr) shouldBe address + addressEncoder.toString(addr) shouldBe address } val p2pk = "3WvsT2Gm4EpsM9Pg18PdY6XyhNNMqXDsvJTbbf6ihLvAmSb7u5RN" @@ -111,7 +111,7 @@ class ScriptApiRouteSpec extends AnyFlatSpec val script = TrueLeaf val tree = ErgoTree.fromProposition(script) - val p2s = ergoAddressEncoder.toString(ergoAddressEncoder.fromProposition(tree).get) + val p2s = addressEncoder.toString(addressEncoder.fromProposition(tree).get) p2s shouldBe "Ms7smJwLGbUAjuWQ" Get(s"$prefix/$suffix/$p2s") ~> route ~> check(assertion(responseAs[Json], p2s)) } @@ -130,9 +130,9 @@ class ScriptApiRouteSpec extends AnyFlatSpec val tree = ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(bs) - val addr = ergoAddressEncoder.fromProposition(tree).get + val addr = addressEncoder.fromProposition(tree).get - ergoAddressEncoder.toString(addr) shouldBe address + addressEncoder.toString(addr) shouldBe address } val p2pk = "3WvsT2Gm4EpsM9Pg18PdY6XyhNNMqXDsvJTbbf6ihLvAmSb7u5RN" @@ -144,7 +144,7 @@ class ScriptApiRouteSpec extends AnyFlatSpec val script = TrueLeaf val tree = ErgoTree.fromProposition(script) - val p2s = ergoAddressEncoder.toString(ergoAddressEncoder.fromProposition(tree).get) + val p2s = addressEncoder.toString(addressEncoder.fromProposition(tree).get) p2s shouldBe "Ms7smJwLGbUAjuWQ" Get(s"$prefix/$suffix/$p2s") ~> route ~> check(assertion(responseAs[Json], p2s)) } diff --git a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala index df998daf35..3aa73d9dc1 100644 --- a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala @@ -46,15 +46,15 @@ class WalletApiRouteSpec extends AnyFlatSpec implicit val requestsHolderEncoder: RequestsHolderEncoder = new RequestsHolderEncoder(ergoSettings) implicit val addressJsonDecoder: Decoder[ErgoAddress] = ErgoAddressJsonEncoder(settings).decoder - val paymentRequest = PaymentRequest(Pay2SAddress(Constants.FalseLeaf), 100L, Seq.empty, Map.empty) - val assetIssueRequest = AssetIssueRequest(Pay2SAddress(Constants.FalseLeaf), None, 100L, "TEST", "Test", 8) + val paymentRequest = PaymentRequest(Pay2SAddress(Constants.FalseLeaf)(addressEncoder), 100L, Seq.empty, Map.empty) + val assetIssueRequest = AssetIssueRequest(Pay2SAddress(Constants.FalseLeaf)(addressEncoder), None, 100L, "TEST", "Test", 8) val requestsHolder = RequestsHolder( (0 to 10).flatMap(_ => Seq(paymentRequest, assetIssueRequest)), Some(10000L), Seq.empty, Seq.empty, minerRewardDelay = 720 - ) + )(addressEncoder) it should "generate arbitrary transaction" in { diff --git a/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala b/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala index e19c32782c..09974da381 100644 --- a/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala +++ b/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala @@ -49,7 +49,7 @@ class MempoolAuditorSpec extends AnyFlatSpec with NodeViewTestOps with ErgoTestH } .get applyBlock(genesis) shouldBe 'success - getRootHash shouldBe Algos.encode(wusAfterGenesis.rootHash) + getRootHash shouldBe Algos.encode(wusAfterGenesis.rootDigest) val boxes = ErgoState.newBoxes(genesis.transactions).find(_.ergoTree == Constants.TrueLeaf) boxes.nonEmpty shouldBe true diff --git a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala index c65d89c64c..0bc8866630 100644 --- a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala @@ -52,7 +52,7 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event val testProbe = new TestProbe(system) system.eventStream.subscribe(testProbe.ref, newBlockSignal) - val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings, timeProvider) + val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(viewHolderRef) val candidateGenerator: ActorRef = @@ -60,7 +60,6 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event defaultMinerSecret.publicImage, readersHolderRef, viewHolderRef, - timeProvider, defaultSettings ) ErgoMiningThread(defaultSettings, candidateGenerator, defaultMinerSecret.w) @@ -75,7 +74,7 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event val testProbe = new TestProbe(system) system.eventStream.subscribe(testProbe.ref, newBlockSignal) - val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings, timeProvider) + val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(viewHolderRef) val candidateGenerator: ActorRef = @@ -83,7 +82,6 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event defaultMinerSecret.publicImage, readersHolderRef, viewHolderRef, - timeProvider, defaultSettings ) @@ -119,7 +117,7 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event val testProbe = new TestProbe(system) system.eventStream.subscribe(testProbe.ref, newBlockSignal) - val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings, timeProvider) + val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(viewHolderRef) val candidateGenerator: ActorRef = @@ -127,7 +125,6 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event defaultMinerSecret.publicImage, readersHolderRef, viewHolderRef, - timeProvider, defaultSettings ) @@ -159,7 +156,7 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event ) { val testProbe = new TestProbe(system) system.eventStream.subscribe(testProbe.ref, newBlockSignal) - val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings, timeProvider) + val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(viewHolderRef) val candidateGenerator: ActorRef = @@ -167,7 +164,6 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event defaultMinerSecret.publicImage, readersHolderRef, viewHolderRef, - timeProvider, defaultSettings ) diff --git a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala index 2dd69676ac..d17b3da442 100644 --- a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala @@ -68,14 +68,13 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen } complexScript.complexity shouldBe 28077 - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) val minerRef: ActorRef = ErgoMiner( ergoSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) expectNoMessage(1 second) @@ -145,7 +144,7 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen val testProbe = new TestProbe(system) system.eventStream.subscribe(testProbe.ref, newBlockSignal) - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) expectNoMessage(1 second) val r: Readers = requestReaders @@ -155,7 +154,6 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen ergoSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) minerRef ! StartMining @@ -220,14 +218,13 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen system.eventStream.subscribe(testProbe.ref, newBlockSignal) val ergoSettings: ErgoSettings = defaultSettings.copy(directory = createTempDir.getAbsolutePath) - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) val minerRef: ActorRef = ErgoMiner( ergoSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) expectNoMessage(1 second) @@ -296,14 +293,13 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen it should "prepare external candidate" in new TestKit(ActorSystem()) { val ergoSettings: ErgoSettings = defaultSettings.copy(directory = createTempDir.getAbsolutePath) - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) def minerRef: ActorRef = ErgoMiner( ergoSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) @@ -320,14 +316,13 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen system.eventStream.subscribe(testProbe.ref, newBlockSignal) val ergoSettings: ErgoSettings = defaultSettings.copy(directory = createTempDir.getAbsolutePath) - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) val minerRef: ActorRef = ErgoMiner( ergoSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) expectNoMessage(1 second) @@ -399,14 +394,13 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen empty.copy(nodeSettings = nodeSettings, chainSettings = chainSettings, directory = createTempDir.getAbsolutePath) } - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(forkSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(forkSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) val minerRef: ActorRef = ErgoMiner( forkSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) diff --git a/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala b/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala index 4a1f08a0f5..16a2389dfa 100644 --- a/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala +++ b/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala @@ -14,7 +14,6 @@ import org.ergoplatform.wallet.protocol.context.{InputContext, TransactionContex import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, Input} import org.scalacheck.Gen import scalan.util.BenchmarkUtil -import scorex.core.transaction.Transaction import scorex.crypto.authds.ADKey import scorex.crypto.hash.{Blake2b256, Digest32} import scorex.db.ByteArrayWrapper @@ -240,7 +239,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { val validity = tx.statefulValidity(from, emptyDataBoxes, emptyStateContext) validity.isSuccess shouldBe false val e = validity.failed.get - e.getMessage should startWith(ValidationRules.errorMessage(ValidationRules.txScriptValidation, "", emptyModifierId, Transaction.ModifierTypeId)) + e.getMessage should startWith(ValidationRules.errorMessage(ValidationRules.txScriptValidation, "", emptyModifierId, ErgoTransaction.modifierTypeId)) } } @@ -317,7 +316,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { } val txMod = tx.copy(inputs = inputsPointers, outputCandidates = out) val validFailure = txMod.statefulValidity(in, emptyDataBoxes, emptyStateContext) - validFailure.failed.get.getMessage should startWith(ValidationRules.errorMessage(txAssetsInOneBox, "", emptyModifierId, Transaction.ModifierTypeId).take(30)) + validFailure.failed.get.getMessage should startWith(ValidationRules.errorMessage(txAssetsInOneBox, "", emptyModifierId, ErgoTransaction.modifierTypeId).take(30)) } property("transaction with too many inputs should be rejected") { @@ -347,7 +346,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { assert(time0 <= Timeout) val cause = validity.failed.get.getMessage - cause should startWith(ValidationRules.errorMessage(bsBlockTransactionsCost, "", emptyModifierId, Transaction.ModifierTypeId).take(30)) + cause should startWith(ValidationRules.errorMessage(bsBlockTransactionsCost, "", emptyModifierId, ErgoTransaction.modifierTypeId).take(30)) //check that spam transaction validation with no cost limit is indeed taking too much time import Parameters._ @@ -535,7 +534,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { val txFailure = tx.statefulValidity(boxes, IndexedSeq.empty, ctx3) txFailure.isSuccess shouldBe false val cause = txFailure.toEither.left.get.getMessage - val expectedMessage = ValidationRules.errorMessage(ValidationRules.txMonotonicHeight, "", emptyModifierId, Transaction.ModifierTypeId) + val expectedMessage = ValidationRules.errorMessage(ValidationRules.txMonotonicHeight, "", emptyModifierId, ErgoTransaction.modifierTypeId) cause should startWith(expectedMessage) } } diff --git a/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala b/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala index 5e8349e802..8fd26ff081 100644 --- a/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala +++ b/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala @@ -24,7 +24,6 @@ import scorex.core.network.message._ import scorex.core.network.peer.PeerInfo import scorex.core.network.{ConnectedPeer, DeliveryTracker} import scorex.core.serialization.ScorexSerializer -import scorex.core.utils.NetworkTimeProvider import scorex.testkit.utils.AkkaFixture import scala.concurrent.duration.{Duration, _} @@ -54,13 +53,12 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc } } - class NodeViewHolderMock extends ErgoNodeViewHolder[UtxoState](settings, timeProvider) + class NodeViewHolderMock extends ErgoNodeViewHolder[UtxoState](settings) class SynchronizerMock(networkControllerRef: ActorRef, viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, - timeProvider: NetworkTimeProvider, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker) (implicit ec: ExecutionContext) extends ErgoNodeViewSynchronizer( @@ -68,15 +66,14 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc viewHolderRef, syncInfoSpec, settings, - timeProvider, syncTracker, deliveryTracker)(ec) { protected def broadcastInvForNewModifier(mod: PersistentNodeViewModifier): Unit = { mod match { - case fb: ErgoFullBlock if fb.header.isNew(timeProvider, 1.hour) => + case fb: ErgoFullBlock if fb.header.isNew(1.hour) => fb.toSeq.foreach(s => broadcastModifierInv(s)) - case h: Header if h.isNew(timeProvider, 1.hour) => + case h: Header if h.isNew(1.hour) => broadcastModifierInv(h) case _ => } @@ -132,11 +129,10 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc val settings = ErgoSettings.read() val pool = ErgoMemPool.empty(settings) implicit val ec: ExecutionContextExecutor = system.dispatcher - val tp = new NetworkTimeProvider(settings.scorexSettings.ntp) val ncProbe = TestProbe("NetworkControllerProbe") val pchProbe = TestProbe("PeerHandlerProbe") val eventListener = TestProbe("EventListener") - val syncTracker = ErgoSyncTracker(settings.scorexSettings.network, timeProvider) + val syncTracker = ErgoSyncTracker(settings.scorexSettings.network) val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(settings) // each test should always start with empty history @@ -149,7 +145,6 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc nodeViewHolderMockRef, ErgoSyncInfoMessageSpec, settings, - tp, syncTracker, deliveryTracker) )) @@ -157,7 +152,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc @SuppressWarnings(Array("org.wartremover.warts.OptionPartial")) val tx = validErgoTransactionGenTemplate(0, 0).sample.get._2 - val peerInfo = PeerInfo(defaultPeerSpec, timeProvider.time()) + val peerInfo = PeerInfo(defaultPeerSpec, System.currentTimeMillis()) @SuppressWarnings(Array("org.wartremover.warts.OptionPartial")) val p: ConnectedPeer = ConnectedPeer( connectionIdGen.sample.get, @@ -180,7 +175,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc implicit val ec: ExecutionContextExecutor = system.dispatcher val ncProbe = TestProbe("NetworkControllerProbe") val pchProbe = TestProbe("PeerHandlerProbe") - val syncTracker = ErgoSyncTracker(settings.scorexSettings.network, timeProvider) + val syncTracker = ErgoSyncTracker(settings.scorexSettings.network) val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(settings) // each test should always start with empty history @@ -193,12 +188,11 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc nodeViewHolderMockRef, ErgoSyncInfoMessageSpec, settings, - timeProvider, syncTracker, deliveryTracker) )) - val peerInfo = PeerInfo(defaultPeerSpec, timeProvider.time()) + val peerInfo = PeerInfo(defaultPeerSpec, System.currentTimeMillis()) @SuppressWarnings(Array("org.wartremover.warts.OptionPartial")) val peer: ConnectedPeer = ConnectedPeer( connectionIdGen.sample.get, @@ -254,7 +248,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc implicit val patienceConfig: PatienceConfig = PatienceConfig(5.second, 100.millis) // we generate and apply existing base chain - val hhistory = ErgoHistory.readOrGenerate(settings, timeProvider) + val hhistory = ErgoHistory.readOrGenerate(settings)(null) val baseChain = genHeaderChain(_.size > 4, None, hhistory.difficultyCalculator, None, false) baseChain.headers.foreach(hhistory.append) val bestHeaderOpt = hhistory.bestHeaderOpt @@ -290,7 +284,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc } eventually { // test whether applied header was actually persisted to history - val hist = ErgoHistory.readOrGenerate(settings, timeProvider) + val hist = ErgoHistory.readOrGenerate(settings)(null) hist.bestHeaderIdOpt.get shouldBe appliedHeader.id } } @@ -313,7 +307,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc // we generate fork of two headers, starting from the parent of the best header // so the depth of the rollback is 1, and the fork bypasses the best chain by 1 header - val hhistory = ErgoHistory.readOrGenerate(settings, timeProvider) + val hhistory = ErgoHistory.readOrGenerate(settings)(null) val newHeaders = genHeaderChain(2, hhistory, diffBitsOpt = None, useRealTs = false).headers val newHistory = newHeaders.foldLeft(hhistory) { case (hist, header) => hist.append(header).get._1 } val parentOpt = newHistory.lastHeaders(2).headOption @@ -354,7 +348,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc deliveryTracker.reset() - val hist = ErgoHistory.readOrGenerate(settings, timeProvider) + val hist = ErgoHistory.readOrGenerate(settings)(null) // generate smaller fork that is going to be reverted after applying a bigger fork val smallFork = genChain(4, hist) diff --git a/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala b/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala index 02872c3449..e0d118fcdc 100644 --- a/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala +++ b/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala @@ -11,7 +11,7 @@ class ErgoSyncTrackerSpecification extends ErgoPropertyTest { val peerInfo = PeerInfo(defaultPeerSpec, time, Some(Incoming)) val cid = ConnectionId(inetAddr1, inetAddr2, Incoming) val connectedPeer = ConnectedPeer(cid, handlerRef = null, lastMessage = 5L, Some(peerInfo)) - val syncTracker = ErgoSyncTracker(settings.scorexSettings.network, timeProvider) + val syncTracker = ErgoSyncTracker(settings.scorexSettings.network) val height = 1000 // add peer to sync diff --git a/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala index d4290b4ba6..a059e416e9 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala @@ -1,6 +1,6 @@ package org.ergoplatform.nodeView.history -import org.ergoplatform.modifiers.NonHeaderBlockSection +import org.ergoplatform.modifiers.{BlockSection, NonHeaderBlockSection} import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.Header @@ -79,10 +79,10 @@ class BlockSectionValidationSpecification extends HistoryTestHelpers { // should not be able to apply if corresponding header is marked as invalid history.applicableTry(section) shouldBe 'success - history.historyStorage.insert(Seq(history.validityKey(header.id) -> Array(0.toByte)), Seq.empty).get + history.historyStorage.insert(Array(history.validityKey(header.id) -> Array(0.toByte)), Array.empty[BlockSection]).get history.isSemanticallyValid(header.id) shouldBe ModifierSemanticValidity.Invalid history.applicableTry(section) shouldBe 'failure - history.historyStorage.insert(Seq(history.validityKey(header.id) -> Array(1.toByte)), Seq.empty).get + history.historyStorage.insert(Array(history.validityKey(header.id) -> Array(1.toByte)), Array.empty[BlockSection]).get // should not be able to apply if already in history history.applicableTry(section) shouldBe 'success diff --git a/src/test/scala/org/ergoplatform/nodeView/history/NonVerifyADHistorySpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/NonVerifyADHistorySpecification.scala index 43e3560112..15dd345048 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/NonVerifyADHistorySpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/NonVerifyADHistorySpecification.scala @@ -5,7 +5,6 @@ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.history.popow.NipopowAlgos import org.ergoplatform.modifiers.history.HeaderChain -import org.ergoplatform.modifiers.state.UTXOSnapshotChunk import org.ergoplatform.nodeView.state.StateType import org.ergoplatform.settings.Algos import org.ergoplatform.utils.HistoryTestHelpers @@ -22,15 +21,6 @@ class NonVerifyADHistorySpecification extends HistoryTestHelpers { private lazy val popowHistory = ensureMinimalHeight(genHistory(), 100) - ignore("Should apply UTXOSnapshotChunks") { - forAll(randomUTXOSnapshotChunkGen) { snapshot: UTXOSnapshotChunk => - popowHistory.applicable(snapshot) shouldBe true - val processInfo = popowHistory.append(snapshot).get._2 - processInfo.toApply shouldEqual Some(snapshot) - popowHistory.applicable(snapshot) shouldBe false - } - } - property("Should calculate difficulty correctly") { val epochLength = 3 val useLastEpochs = 3 diff --git a/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala index 4ef3e68b54..30908ceb1a 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala @@ -1,18 +1,17 @@ package org.ergoplatform.nodeView.history -import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.modifiers.{ErgoFullBlock, NetworkObjectTypeId} import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.HeaderSerializer -import org.ergoplatform.nodeView.history.storage.modifierprocessors.{FullBlockProcessor, ToDownloadProcessor} +import org.ergoplatform.nodeView.history.storage.modifierprocessors.FullBlockProcessor import org.ergoplatform.nodeView.state.StateType import org.ergoplatform.settings.Algos import org.ergoplatform.utils.HistoryTestHelpers -import scorex.core.ModifierTypeId import scorex.core.consensus.ProgressInfo class VerifyNonADHistorySpecification extends HistoryTestHelpers { - import ToDownloadProcessor._ + import org.ergoplatform.nodeView.history.storage.modifierprocessors.ToDownloadProcessor._ private def genHistory() = generateHistory(verifyTransactions = true, StateType.Utxo, PoPoWBootstrap = false, BlocksToKeep) @@ -118,7 +117,7 @@ class VerifyNonADHistorySpecification extends HistoryTestHelpers { val missedChain = chain.tail.toList val missedBS = missedChain.flatMap { fb => Seq((BlockTransactions.modifierTypeId, fb.blockTransactions.encodedId), (Extension.modifierTypeId, fb.extension.encodedId)) - }.foldLeft(Map.empty[ModifierTypeId, Seq[String]]) { case (newAcc, (mType, mId)) => + }.foldLeft(Map.empty[NetworkObjectTypeId.Value, Seq[String]]) { case (newAcc, (mType, mId)) => newAcc.adjust(mType)(_.fold(Seq(mId))(_ :+ mId)) } diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala new file mode 100644 index 0000000000..c51c72c9d1 --- /dev/null +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala @@ -0,0 +1,291 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, ErgoBox, ErgoBoxCandidate, ErgoScriptPredef, P2PKAddress, UnsignedInput} +import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.mining.difficulty.RequiredDifficulty +import org.ergoplatform.mining.{AutolykosPowScheme, CandidateBlock, CandidateGenerator} +import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionCandidate} +import org.ergoplatform.modifiers.history.header.Header +import org.ergoplatform.modifiers.history.popow.NipopowAlgos +import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction} +import org.ergoplatform.nodeView.history.ErgoHistory +import org.ergoplatform.nodeView.mempool.ErgoMemPool.SortingOption +import org.ergoplatform.nodeView.state.{ErgoState, ErgoStateContext, StateConstants, StateType, UtxoState, UtxoStateReader} +import org.ergoplatform.settings.{ErgoSettings, NetworkType, NodeConfigurationSettings} +import org.ergoplatform.utils.{ErgoPropertyTest, ErgoTestHelpers, HistoryTestHelpers} +import scorex.util.{ModifierId, bytesToId} +import sigmastate.Values +import sigmastate.basics.DLogProtocol.ProveDlog +import spire.implicits.cfor + +import java.io.File +import scala.annotation.tailrec +import scala.collection.mutable +import scala.concurrent.duration.{DurationInt, FiniteDuration} +import scala.util.Try + +class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase with HistoryTestHelpers { + + override protected val saveLimit: Int = 1 // save every block + override protected implicit val addressEncoder: ErgoAddressEncoder = initSettings.chainSettings.addressEncoder + + val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(StateType.Utxo, verifyTransactions = true, + -1, poPoWBootstrap = false, ChainGenerator.minimalSuffix, mining = false, ChainGenerator.txCostLimit, ChainGenerator.txSizeLimit, useExternalMiner = false, + internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false, + 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 20, + 1000000, 100, adProofsSuffixLength = 112 * 1024, extraIndex = false) + + val HEIGHT: Int = 30 + val BRANCHPOINT: Int = HEIGHT / 2 + + property("extra indexer rollback") { + + val dir: File = createTempDir + dir.mkdirs() + + val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, initSettings.chainSettings, + nodeSettings, settings.scorexSettings, settings.walletSettings, settings.cacheSettings) + + _history = ErgoHistory.readOrGenerate(fullHistorySettings)(null) + + ChainGenerator.generate(HEIGHT, dir)(_history) + + run() + + val txIndexBefore = globalTxIndex + val boxIndexBefore = globalBoxIndex + + var txsIndexed: Int = 0 + var boxesIndexed: Int = 0 + + // manually count balances + val addresses: mutable.HashMap[ErgoAddress,Long] = mutable.HashMap[ErgoAddress,Long]() + cfor(1)(_ <= BRANCHPOINT, _ + 1) { i => + _history.getReader.bestBlockTransactionsAt(i).get.txs.foreach(tx => { txsIndexed += 1 + if(i != 1) { + tx.inputs.foreach(input => { + val iEb: IndexedErgoBox = _history.getReader.typedExtraIndexById[IndexedErgoBox](bytesToId(input.boxId)).get + val address: ErgoAddress = ExtraIndexer.getAddress(iEb.box.ergoTree) + addresses.put(address, addresses(address) - iEb.box.value) + }) + } + tx.outputs.foreach(output => { boxesIndexed += 1 + val address: ErgoAddress = addressEncoder.fromProposition(output.ergoTree).get + addresses.put(address, addresses.getOrElse[Long](address, 0) + output.value) + }) + }) + } + + removeAfter(BRANCHPOINT) + + var mismatches: Int = 0 + + addresses.foreach(e => { + _history.getReader.typedExtraIndexById[IndexedErgoAddress](bytesToId(IndexedErgoAddressSerializer.hashErgoTree(e._1.script))) match { + case Some(iEa) => + if(iEa.balanceInfo.get.nanoErgs != e._2) { + mismatches += 1 + System.err.println(s"Address ${e._1.toString} has ${iEa.balanceInfo.get.nanoErgs / 1000000000}ERG, ${e._2 / 1000000000}ERG expected") + } + case None => + if(e._2 != 0) { + mismatches += 1 + System.err.println(s"Address ${e._1.toString} should exist, but was not found") + } + } + }) + + // indexnumbers + globalTxIndex shouldBe txsIndexed + globalBoxIndex shouldBe boxesIndexed + + // txs + cfor(0)(_ < txIndexBefore, _ + 1) {txNum => + val txOpt = history.typedExtraIndexById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(txNum))) + if(txNum < globalTxIndex) + txOpt shouldNot be(empty) + else + txOpt shouldBe None + } + + // boxes + cfor(0)(_ < boxIndexBefore, _ + 1) { boxNum => + val boxOpt = history.typedExtraIndexById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(boxNum))) + if (boxNum < globalBoxIndex) + boxOpt shouldNot be(empty) + else + boxOpt shouldBe None + } + + // balances + mismatches shouldBe 0 + + } + +} + +object ChainGenerator extends ErgoTestHelpers { + + val pow: AutolykosPowScheme = new AutolykosPowScheme(powScheme.k, powScheme.n) + val blockInterval: FiniteDuration = 2.minute + val EmissionTxCost: Long = 20000 + val MinTxAmount: Long = 2000000 + val RewardDelay: Int = initSettings.chainSettings.monetary.minerRewardDelay + val MaxTxsPerBlock: Int = 10 + val minerPk: ProveDlog = defaultProver.hdKeys.head.publicImage + val selfAddressScript: Values.ErgoTree = P2PKAddress(minerPk).script + val minerProp: Values.ErgoTree = ErgoScriptPredef.rewardOutputScript(RewardDelay, minerPk) + val votingEpochLength: Height = votingSettings.votingLength + val protocolVersion: Byte = initSettings.chainSettings.protocolVersion + val minimalSuffix = 2 + val txCostLimit: Height = initSettings.nodeSettings.maxTransactionCost + val txSizeLimit: Height = initSettings.nodeSettings.maxTransactionSize + + var startTime: Long = 0 + + def generate(length: Int, dir: File)(history: ErgoHistory): Unit = { + val stateDir = new File(s"${dir.getAbsolutePath}/state") + stateDir.mkdirs() + val (state, _) = ErgoState.generateGenesisUtxoState(stateDir, StateConstants(initSettings)) + System.out.println(s"Going to generate a chain at ${dir.getAbsolutePath} starting from ${history.bestFullBlockOpt}") + startTime = System.currentTimeMillis() - (blockInterval * (length - 1)).toMillis + val chain = loop(state, None, None, Seq())(history) + System.out.println(s"Chain of length ${chain.length} generated") + history.bestHeaderOpt shouldBe history.bestFullBlockOpt.map(_.header) + history.bestFullBlockOpt.get.id shouldBe chain.last + System.out.println("History was generated successfully") + } + + @tailrec + private def loop(state: UtxoState, + initBox: Option[ErgoBox], + last: Option[Header], + acc: Seq[ModifierId])(history: ErgoHistory): Seq[ModifierId] = { + val time: Long = last.map(_.timestamp + blockInterval.toMillis).getOrElse(startTime) + if (time < System.currentTimeMillis()) { + val (txs, lastOut) = genTransactions(last.map(_.height).getOrElse(ErgoHistory.GenesisHeight), + initBox, state.stateContext) + + val candidate = genCandidate(defaultProver.hdPubKeys.head.key, last, time, txs, state)(history) + val block = proveCandidate(candidate.get) + + history.append(block.header).get + block.blockSections.foreach(s => if (!history.contains(s)) history.append(s).get) + + val outToPassNext = if (last.isEmpty) { + block.transactions.flatMap(_.outputs).find(_.ergoTree == minerProp) + } else { + lastOut + } + + assert(outToPassNext.isDefined) + + log.info( + s"Block ${block.id} with ${block.transactions.size} transactions at height ${block.header.height} generated") + + loop(state.applyModifier(block, None)(_ => ()).get, outToPassNext, Some(block.header), acc :+ block.id)(history) + } else { + acc + } + } + + private def genTransactions(height: Height, + inOpt: Option[ErgoBox], + ctx: ErgoStateContext): (Seq[ErgoTransaction], Option[ErgoBox]) = { + inOpt + .find { bx => + val canUnlock = (bx.creationHeight + RewardDelay <= height) || (bx.ergoTree != minerProp) + canUnlock && bx.ergoTree != initSettings.chainSettings.monetary.emissionBoxProposition && bx.value >= MinTxAmount + } + .map { input => + val qty = MaxTxsPerBlock + val amount = input.value + val outs = (0 until qty).map(_ => new ErgoBoxCandidate(amount, selfAddressScript, height)) + val x = outs + .foldLeft((Seq.empty[ErgoTransaction], input)) { case ((acc, in), out) => + val inputs = IndexedSeq(in) + val unsignedTx = UnsignedErgoTransaction( + inputs.map(_.id).map(id => new UnsignedInput(id)), + IndexedSeq(out) + ) + + defaultProver.sign(unsignedTx, inputs, emptyDataBoxes, ctx) + .fold(_ => acc -> in, tx => (acc :+ ErgoTransaction(tx)) -> unsignedTx.outputs.head) + } + ._1 + (x, Some(x.last.outputs.head)) + } + .getOrElse(Seq.empty -> inOpt) + } + + private def genCandidate(minerPk: ProveDlog, + lastHeaderOpt: Option[Header], + ts: Long, + txsFromPool: Seq[ErgoTransaction], + state: UtxoStateReader)(history: ErgoHistory): Try[CandidateBlock] = Try { + val stateContext = state.stateContext + val nBits: Long = lastHeaderOpt + .map(parent => history.requiredDifficultyAfter(parent)) + .map(d => RequiredDifficulty.encodeCompactBits(d)) + .getOrElse(settings.chainSettings.initialNBits) + + val interlinks = lastHeaderOpt + .flatMap { h => + history.typedModifierById[Extension](h.extensionId) + .flatMap(ext => NipopowAlgos.unpackInterlinks(ext.fields).toOption) + .map(nipopowAlgos.updateInterlinks(h, _)) + } + .getOrElse(Seq.empty) + val interlinksExtension = nipopowAlgos.interlinksToExtension(interlinks) + + val (extensionCandidate, votes: Array[Byte], version: Byte) = lastHeaderOpt.map { header => + val newHeight = header.height + 1 + val currentParams = stateContext.currentParameters + val betterVersion = protocolVersion > header.version + val votingFinishHeight: Option[Height] = currentParams.softForkStartingHeight + .map(_ + votingSettings.votingLength * votingSettings.softForkEpochs) + val forkVotingAllowed = votingFinishHeight.forall(fh => newHeight < fh) + val forkOrdered = settings.votingTargets.softFork != 0 + val voteForFork = betterVersion && forkOrdered && forkVotingAllowed + + if (newHeight % votingEpochLength == 0 && newHeight > 0) { + val (newParams, _) = currentParams.update(newHeight, voteForFork, stateContext.votingData.epochVotes, emptyVSUpdate, votingSettings) + (newParams.toExtensionCandidate ++ interlinksExtension, + newParams.suggestVotes(settings.votingTargets.targets, voteForFork), + newParams.blockVersion) + } else { + (nipopowAlgos.interlinksToExtension(interlinks), + currentParams.vote(settings.votingTargets.targets, stateContext.votingData.epochVotes, voteForFork), + currentParams.blockVersion) + } + }.getOrElse((interlinksExtension, Array(0: Byte, 0: Byte, 0: Byte), Header.InitialVersion)) + + val emissionTxOpt = CandidateGenerator.collectEmission(state, minerPk, emptyStateContext) + val txs = emissionTxOpt.toSeq ++ txsFromPool + + state.proofsForTransactions(txs).map { case (adProof, adDigest) => + CandidateBlock(lastHeaderOpt, version, nBits, adDigest, adProof, txs, ts, extensionCandidate, votes) + } + }.flatten + + @tailrec + private def proveCandidate(candidate: CandidateBlock): ErgoFullBlock = { + log.info(s"Trying to prove block with parent ${candidate.parentOpt.map(_.encodedId)} and timestamp ${candidate.timestamp}") + + pow.proveCandidate(candidate, defaultProver.hdKeys.head.privateInput.w) match { + case Some(fb) => fb + case _ => + val interlinks = candidate.parentOpt + .map(nipopowAlgos.updateInterlinks(_, NipopowAlgos.unpackInterlinks(candidate.extension.fields).get)) + .getOrElse(Seq.empty) + val minerTag = scorex.utils.Random.randomBytes(Extension.FieldKeySize) + proveCandidate { + candidate.copy( + extension = ExtensionCandidate(Seq(Array(0: Byte, 2: Byte) -> minerTag)) ++ nipopowAlgos.interlinksToExtension(interlinks) + ) + } + } + } + +} diff --git a/src/test/scala/org/ergoplatform/nodeView/history/storage/HistoryStorageSpec.scala b/src/test/scala/org/ergoplatform/nodeView/history/storage/HistoryStorageSpec.scala index 6510d46b2f..7020d1fee7 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/storage/HistoryStorageSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/storage/HistoryStorageSpec.scala @@ -1,5 +1,8 @@ package org.ergoplatform.nodeView.history.storage +import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.modifiers.history.ADProofs +import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.settings.Algos import org.ergoplatform.utils.HistoryTestHelpers @@ -12,11 +15,11 @@ class HistoryStorageSpec extends HistoryTestHelpers { val db = HistoryStorage(settings) property("Write Read Remove") { - val headers = Gen.listOfN(20, defaultHeaderGen).sample.get - val modifiers = Gen.listOfN(20, randomADProofsGen).sample.get + val headers: Array[Header] = Gen.listOfN(20, defaultHeaderGen).sample.get.toArray + val modifiers: Array[ADProofs] = Gen.listOfN(20, randomADProofsGen).sample.get.toArray def validityKey(id: ModifierId) = ByteArrayWrapper(Algos.hash("validity".getBytes(ErgoHistory.CharsetName) ++ idToBytes(id))) - val indexes = headers.flatMap(h => Seq(validityKey(h.id) -> Array(1.toByte))) - db.insert(indexes, headers ++ modifiers) shouldBe 'success + val indexes = headers.flatMap(h => Array(validityKey(h.id) -> Array(1.toByte))) + db.insert(indexes, (headers ++ modifiers).asInstanceOf[Array[BlockSection]]) shouldBe 'success headers.forall(h => db.contains(h.id)) shouldBe true modifiers.forall(m => db.contains(m.id)) shouldBe true diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala index faaf40218e..c68d279f50 100644 --- a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala @@ -378,6 +378,20 @@ class ErgoMemPoolSpec extends AnyFlatSpec pool.size shouldBe 0 pool.stats.takenTxns shouldBe (family_depth + 1) * txs.size } + + it should "put not adding transaction twice" in { + val pool = ErgoMemPool.empty(settings).pool + val tx = invalidErgoTransactionGen.sample.get + val now = System.currentTimeMillis() + + val utx1 = new UnconfirmedTransaction(tx, None, now, now, None, None) + val utx2 = new UnconfirmedTransaction(tx, None, now, now, None, None) + val utx3 = new UnconfirmedTransaction(tx, None, now + 1, now + 1, None, None) + val updPool = pool.put(utx1, 100).remove(utx1).put(utx2, 500).put(utx3, 5000) + updPool.size shouldBe 1 + updPool.get(utx3.id).get.lastCheckedTime shouldBe (now + 1) + } + } diff --git a/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala index aa5eae981c..4d670365d1 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala @@ -21,19 +21,19 @@ class DigestStateSpecification extends ErgoPropertyTest { val fb = validFullBlock(parentOpt = None, us, bh) val dir2 = createTempDir - val ds = DigestState.create(Some(us.version), Some(us.rootHash), dir2, stateConstants) + val ds = DigestState.create(Some(us.version), Some(us.rootDigest), dir2, stateConstants) ds.applyModifier(fb, None)(_ => ()) shouldBe 'success ds.close() val state = DigestState.create(None, None, dir2, stateConstants) state.version shouldEqual fb.header.id - state.rootHash shouldEqual fb.header.stateRoot + state.rootDigest shouldEqual fb.header.stateRoot } } property("validate() - valid block") { var (us, bh) = createUtxoState(parameters) - var ds = createDigestState(us.version, us.rootHash, parameters) + var ds = createDigestState(us.version, us.rootDigest, parameters) var parentOpt: Option[ErgoFullBlock] = None forAll { seed: Int => @@ -61,7 +61,7 @@ class DigestStateSpecification extends ErgoPropertyTest { val block = validFullBlock(parentOpt = None, us, bh) block.blockTransactions.transactions.exists(_.dataInputs.nonEmpty) shouldBe true - val ds = createDigestState(us.version, us.rootHash, parameters) + val ds = createDigestState(us.version, us.rootDigest, parameters) ds.applyModifier(block, None)(_ => ()) shouldBe 'success } } @@ -80,7 +80,7 @@ class DigestStateSpecification extends ErgoPropertyTest { val block = validFullBlock(parentOpt = None, us, bh) - val ds = createDigestState(us.version, us.rootHash, parameters) + val ds = createDigestState(us.version, us.rootDigest, parameters) ds.rollbackVersions.size shouldEqual 1 @@ -90,21 +90,21 @@ class DigestStateSpecification extends ErgoPropertyTest { ds2.stateContext.lastHeaders.size shouldEqual 1 - java.util.Arrays.equals(ds2.rootHash, ds.rootHash) shouldBe false + java.util.Arrays.equals(ds2.rootDigest, ds.rootDigest) shouldBe false val ds3 = ds2.rollbackTo(ds.version).get - ds3.rootHash shouldBe ds.rootHash + ds3.rootDigest shouldBe ds.rootDigest ds3.stateContext.lastHeaders.size shouldEqual 0 - ds3.applyModifier(block, None)(_ => ()).get.rootHash shouldBe ds2.rootHash + ds3.applyModifier(block, None)(_ => ()).get.rootDigest shouldBe ds2.rootDigest } } property("validateTransactions() - dataInputs") { forAll(boxesHolderGen) { bh => val us = createUtxoState(bh, parameters) - val ds = createDigestState(us.version, us.rootHash, parameters) + val ds = createDigestState(us.version, us.rootDigest, parameters) // generate 2 independent transactions spending state boxes only val headTx = validTransactionsFromBoxes(1, bh.boxes.take(10).values.toSeq, new RandomWrapper())._1.head diff --git a/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala index caf2623f41..60d49ff68a 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala @@ -20,7 +20,7 @@ class ErgoStateSpecification extends ErgoPropertyTest { property("applyModifier() - double spending") { forAll(boxesHolderGen, Gen.choose(1: Byte, 2: Byte)) { case (bh, version) => val us = createUtxoState(bh, parameters) - val ds = createDigestState(bytesToVersion(Array.fill(32)(100: Byte)), us.rootHash, parameters) + val ds = createDigestState(bytesToVersion(Array.fill(32)(100: Byte)), us.rootDigest, parameters) val validBlock = validFullBlock(None, us, bh) val dsTxs = validBlock.transactions ++ validBlock.transactions @@ -52,7 +52,7 @@ class ErgoStateSpecification extends ErgoPropertyTest { } var (us, bh) = createUtxoState(parameters) - var ds = createDigestState(us.version, us.rootHash, parameters) + var ds = createDigestState(us.version, us.rootDigest, parameters) var lastBlocks: Seq[ErgoFullBlock] = Seq() forAll { seed: Int => val blBh = validFullBlockWithBoxHolder(lastBlocks.headOption, us, bh, new RandomWrapper(Some(seed))) @@ -68,8 +68,8 @@ class ErgoStateSpecification extends ErgoPropertyTest { property("generateGenesisUtxoState & generateGenesisDigestState are compliant") { val settings = ErgoSettings.read(Args.empty) val dir = createTempDir - val rootHash = createUtxoState(parameters)._1.rootHash - val expectedRootHash = ErgoState.generateGenesisDigestState(dir, settings).rootHash + val rootHash = createUtxoState(parameters)._1.rootDigest + val expectedRootHash = ErgoState.generateGenesisDigestState(dir, settings).rootDigest rootHash shouldBe expectedRootHash } diff --git a/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala new file mode 100644 index 0000000000..4f3b0ffe79 --- /dev/null +++ b/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala @@ -0,0 +1,25 @@ +package org.ergoplatform.nodeView.state + +import org.ergoplatform.utils.ErgoPropertyTest + +class SnapshotsInfoSpecification extends ErgoPropertyTest { + + property("makeEmpty / nonEmpty / withNewManifest") { + val empty = SnapshotsInfo.empty + empty.nonEmpty shouldBe false + + val h = 10 + val d = digest32Gen.sample.get + val nonEmpty = empty.withNewManifest(h, d) + nonEmpty.nonEmpty shouldBe true + nonEmpty.availableManifests(h).sameElements(d) shouldBe true + + val h2 = 20 + val d2 = digest32Gen.sample.get + val ne2 = nonEmpty.withNewManifest(h2, d2) + nonEmpty.availableManifests.size shouldBe 1 + ne2.availableManifests(h).sameElements(d) shouldBe true + ne2.availableManifests(h2).sameElements(d2) shouldBe true + } + +} diff --git a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala index 8b16083d58..205caa634a 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala @@ -495,7 +495,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera bh.sortedBoxes.foreach(box => us.boxById(box.id) should not be None) val genesis = validFullBlock(parentOpt = None, us, bh) val wusAfterGenesis = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get - wusAfterGenesis.rootHash shouldEqual genesis.header.stateRoot + wusAfterGenesis.rootDigest shouldEqual genesis.header.stateRoot val (finalState: WrappedUtxoState, chain: Seq[ErgoFullBlock]) = (0 until depth) .foldLeft((wusAfterGenesis, Seq(genesis))) { (sb, _) => @@ -503,17 +503,17 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera val block = validFullBlock(parentOpt = Some(sb._2.last), state) (state.applyModifier(block)(_ => ()).get, sb._2 ++ Seq(block)) } - val finalRoot = finalState.rootHash + val finalRoot = finalState.rootDigest finalRoot shouldEqual chain.last.header.stateRoot val rollbackedState = finalState.rollbackTo(idToVersion(genesis.id)).get - rollbackedState.rootHash shouldEqual genesis.header.stateRoot + rollbackedState.rootDigest shouldEqual genesis.header.stateRoot val finalState2: WrappedUtxoState = chain.tail.foldLeft(rollbackedState) { (state, block) => state.applyModifier(block)(_ => ()).get } - finalState2.rootHash shouldEqual finalRoot + finalState2.rootDigest shouldEqual finalRoot } } } diff --git a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala index d9c8abc729..fe1933fad9 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala @@ -12,7 +12,7 @@ import scala.util.Try class WrappedDigestState(val digestState: DigestState, val wrappedUtxoState: WrappedUtxoState, val settings: ErgoSettings) - extends DigestState(digestState.version, digestState.rootHash, digestState.store, settings) { + extends DigestState(digestState.version, digestState.rootDigest, digestState.store, settings) { override def applyModifier(mod: BlockSection, estimatedTip: Option[Height]) (generate: LocallyGeneratedModifier => Unit): Try[WrappedDigestState] = { diff --git a/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala b/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala index e9f8471729..f3b89434a5 100644 --- a/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala @@ -30,18 +30,18 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi // too big chain update delay val notAcceptableDelay = System.currentTimeMillis() - (initSettings.nodeSettings.acceptableChainUpdateDelay.toMillis + 100) val invalidProgress = ChainProgress(block, 2, 3, notAcceptableDelay) - ErgoNodeViewHolder.checkChainIsHealthy(invalidProgress, history, timeProvider, initSettings).isInstanceOf[ChainIsStuck] shouldBe true + ErgoNodeViewHolder.checkChainIsHealthy(invalidProgress, history, initSettings).isInstanceOf[ChainIsStuck] shouldBe true // acceptable chain update delay val acceptableDelay = System.currentTimeMillis() - 5 val validProgress = ChainProgress(block, 2, 3, acceptableDelay) - ErgoNodeViewHolder.checkChainIsHealthy(validProgress, history, timeProvider, initSettings) shouldBe ChainIsHealthy + ErgoNodeViewHolder.checkChainIsHealthy(validProgress, history, initSettings) shouldBe ChainIsHealthy } private val t1 = TestCase("check genesis state") { fixture => import fixture._ - getCurrentState.rootHash shouldBe getGenesisStateDigest + getCurrentState.rootDigest shouldBe getGenesisStateDigest } private val t2 = TestCase("check history after genesis") { fixture => @@ -171,7 +171,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi applyBlock(block) shouldBe 'success getBestHeaderOpt shouldBe Some(block.header) if (verifyTransactions) { - getRootHash shouldBe Algos.encode(wusAfterBlock.rootHash) + getRootHash shouldBe Algos.encode(wusAfterBlock.rootDigest) } getBestHeaderOpt shouldBe Some(block.header) @@ -183,7 +183,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi applyBlock(brokenBlock2) shouldBe 'success getBestFullBlockOpt shouldBe Some(block) - getRootHash shouldBe Algos.encode(wusAfterBlock.rootHash) + getRootHash shouldBe Algos.encode(wusAfterBlock.rootDigest) getBestHeaderOpt shouldBe Some(block.header) } } @@ -218,7 +218,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi }.get applyBlock(genesis) shouldBe 'success - getRootHash shouldBe Algos.encode(wusAfterGenesis.rootHash) + getRootHash shouldBe Algos.encode(wusAfterGenesis.rootDigest) val chain1block1 = validFullBlock(Some(genesis), wusAfterGenesis) val expectedBestFullBlockOpt = if (verifyTransactions) Some(chain1block1) else None @@ -233,7 +233,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi val wusChain2Block1 = wusAfterGenesis.applyModifier(chain2block1)(mod => nodeViewHolderRef ! mod).get val chain2block2 = validFullBlock(Some(chain2block1), wusChain2Block1) - chain2block1.header.stateRoot shouldEqual wusChain2Block1.rootHash + chain2block1.header.stateRoot shouldEqual wusChain2Block1.rootDigest applyBlock(chain2block2) shouldBe 'success if (verifyTransactions) { @@ -292,7 +292,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi }.get applyBlock(genesis) shouldBe 'success - getRootHash shouldBe Algos.encode(wusAfterGenesis.rootHash) + getRootHash shouldBe Algos.encode(wusAfterGenesis.rootDigest) val chain2block1 = validFullBlock(Some(genesis), wusAfterGenesis) val wusChain2Block1 = wusAfterGenesis.applyModifier(chain2block1)(mod => nodeViewHolderRef ! mod).get @@ -456,7 +456,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi val wusChain2Block1 = wusGenesis.applyModifier(chain2block1)(mod => nodeViewHolderRef ! mod).get val chain2block2 = validFullBlock(Some(chain2block1), wusChain2Block1) - chain2block1.header.stateRoot shouldEqual wusChain2Block1.rootHash + chain2block1.header.stateRoot shouldEqual wusChain2Block1.rootDigest applyBlock(chain2block2) shouldBe 'success if (verifyTransactions) { @@ -508,13 +508,13 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi if (verifyTransactions) { - val initDigest = getCurrentState.rootHash + val initDigest = getCurrentState.rootDigest applyBlock(invalidBlock) shouldBe 'success getBestFullBlockOpt shouldBe None getBestHeaderOpt shouldBe None - getCurrentState.rootHash shouldEqual initDigest + getCurrentState.rootDigest shouldEqual initDigest } } diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala index 552e0bf270..a78cb1dd4d 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala @@ -27,7 +27,7 @@ class WalletStorageSpec val bytes = DerivationPathSerializer.toBytes(path) acc ++ Ints.toByteArray(bytes.length) ++ bytes } - store.insert(Seq(SecretPathsKey -> toInsert)).get + store.insert(Array(SecretPathsKey -> toInsert)).get } forAll(Gen.nonEmptyListOf(derivationPathGen)) { paths => diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala index 0020660acf..e4d9de0836 100644 --- a/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala +++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala @@ -16,7 +16,6 @@ import org.ergoplatform.settings.Constants.HashLength import org.ergoplatform.utils.{ErgoTestHelpers, HistoryTestHelpers} import org.scalacheck.Gen import scorex.core.network.DeliveryTracker -import scorex.core.utils.NetworkTimeProvider import scorex.core.{PersistentNodeViewModifier, bytesToId} import scorex.crypto.authds.ADDigest import scorex.crypto.hash.{Blake2b256, Digest32} @@ -47,7 +46,7 @@ trait ErgoSanity[ST <: ErgoState[ST]] extends HistoryTests override lazy val memPoolGenerator: Gen[MPool] = emptyMemPoolGen override def syntacticallyValidModifier(history: HT): Header = { - val bestTimestamp = history.bestHeaderOpt.map(_.timestamp + 1).getOrElse(timeProvider.time()) + val bestTimestamp = history.bestHeaderOpt.map(_.timestamp + 1).getOrElse(System.currentTimeMillis()) powScheme.prove( history.bestHeaderOpt, @@ -56,7 +55,7 @@ trait ErgoSanity[ST <: ErgoState[ST]] extends HistoryTests ADDigest @@ Array.fill(HashLength + 1)(0.toByte), Digest32 @@ Array.fill(HashLength)(0.toByte), Digest32 @@ Array.fill(HashLength)(0.toByte), - Math.max(timeProvider.time(), bestTimestamp), + Math.max(System.currentTimeMillis(), bestTimestamp), Digest32 @@ Array.fill(HashLength)(0.toByte), Array.fill(3)(0: Byte), defaultMinerSecretNumber @@ -92,7 +91,6 @@ trait ErgoSanity[ST <: ErgoState[ST]] extends HistoryTests viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, - timeProvider: NetworkTimeProvider, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker) (implicit ec: ExecutionContext) extends ErgoNodeViewSynchronizer( @@ -100,15 +98,14 @@ trait ErgoSanity[ST <: ErgoState[ST]] extends HistoryTests viewHolderRef, syncInfoSpec, settings, - timeProvider, syncTracker, deliveryTracker)(ec) { protected def broadcastInvForNewModifier(mod: PersistentNodeViewModifier): Unit = { mod match { - case fb: ErgoFullBlock if fb.header.isNew(timeProvider, 1.hour) => + case fb: ErgoFullBlock if fb.header.isNew(1.hour) => fb.toSeq.foreach(s => broadcastModifierInv(s)) - case h: Header if h.isNew(timeProvider, 1.hour) => + case h: Header if h.isNew(1.hour) => broadcastModifierInv(h) case _ => } diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala index ef617beb32..9616dd81cf 100644 --- a/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala +++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala @@ -18,7 +18,6 @@ import scorex.core.idToBytes import scorex.core.network.{ConnectedPeer, DeliveryTracker} import scorex.core.network.peer.PeerInfo import scorex.core.serialization.ScorexSerializer -import scorex.core.utils.NetworkTimeProvider import scala.concurrent.ExecutionContextExecutor @@ -29,7 +28,7 @@ class ErgoSanityDigest extends ErgoSanity[DIGEST_ST] { override val stateGen: Gen[WrappedDigestState] = { boxesHolderGen.map(WrappedUtxoState(_, createTempDir, None, parameters, settings)).map { wus => - val digestState = DigestState.create(Some(wus.version), Some(wus.rootHash), createTempDir, stateConstants) + val digestState = DigestState.create(Some(wus.version), Some(wus.rootDigest), createTempDir, stateConstants) new WrappedDigestState(digestState, wus, settings) } } @@ -64,12 +63,11 @@ class ErgoSanityDigest extends ErgoSanity[DIGEST_ST] { val v = h.bestFullBlockIdOpt.orElse(h.bestHeaderIdOpt).get s.store.update(idToBytes(v), Seq(), Seq()).get implicit val ec: ExecutionContextExecutor = system.dispatcher - val tp = new NetworkTimeProvider(settings.scorexSettings.ntp) val ncProbe = TestProbe("NetworkControllerProbe") val vhProbe = TestProbe("ViewHolderProbe") val pchProbe = TestProbe("PeerHandlerProbe") val eventListener = TestProbe("EventListener") - val syncTracker = ErgoSyncTracker(settings.scorexSettings.network, timeProvider) + val syncTracker = ErgoSyncTracker(settings.scorexSettings.network) val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(settings) val ref = system.actorOf(Props( new SyncronizerMock( @@ -77,7 +75,6 @@ class ErgoSanityDigest extends ErgoSanity[DIGEST_ST] { vhProbe.ref, ErgoSyncInfoMessageSpec, settings, - tp, syncTracker, deliveryTracker ) diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala index 558d2bec8b..25c5862f33 100644 --- a/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala +++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala @@ -18,7 +18,6 @@ import org.scalacheck.Gen import scorex.core.network.{ConnectedPeer, DeliveryTracker} import scorex.core.network.peer.PeerInfo import scorex.core.serialization.ScorexSerializer -import scorex.core.utils.NetworkTimeProvider import scala.concurrent.ExecutionContextExecutor @@ -59,12 +58,11 @@ class ErgoSanityUTXO extends ErgoSanity[UTXO_ST] with ErgoTestHelpers { val settings = ErgoSettings.read() val pool = ErgoMemPool.empty(settings) implicit val ec: ExecutionContextExecutor = system.dispatcher - val tp = new NetworkTimeProvider(settings.scorexSettings.ntp) val ncProbe = TestProbe("NetworkControllerProbe") val vhProbe = TestProbe("ViewHolderProbe") val pchProbe = TestProbe("PeerHandlerProbe") val eventListener = TestProbe("EventListener") - val syncTracker = ErgoSyncTracker(settings.scorexSettings.network, timeProvider) + val syncTracker = ErgoSyncTracker(settings.scorexSettings.network) val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(settings) val ref = system.actorOf(Props( new SyncronizerMock( @@ -72,7 +70,6 @@ class ErgoSanityUTXO extends ErgoSanity[UTXO_ST] with ErgoTestHelpers { vhProbe.ref, ErgoSyncInfoMessageSpec, settings, - tp, syncTracker, deliveryTracker) )) @@ -81,7 +78,7 @@ class ErgoSanityUTXO extends ErgoSanity[UTXO_ST] with ErgoTestHelpers { val tx = validErgoTransactionGenTemplate(minAssets = 0, maxAssets = 0).sample.get._2 - val peerInfo = PeerInfo(defaultPeerSpec, timeProvider.time()) + val peerInfo = PeerInfo(defaultPeerSpec, System.currentTimeMillis()) @SuppressWarnings(Array("org.wartremover.warts.OptionPartial")) val p: ConnectedPeer = ConnectedPeer( connectionIdGen.sample.get, diff --git a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala index 5f8cf1075c..b01d8e5a32 100644 --- a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala +++ b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala @@ -42,11 +42,12 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { rebroadcastCount = 3, minimalFeeAmount = 0, headerChainDiff = 100, - adProofsSuffixLength = 112*1024 + adProofsSuffixLength = 112*1024, + extraIndex = false ) settings.cacheSettings shouldBe CacheSettings( HistoryCacheSettings( - 12, 100, 1000 + 12, 500, 100, 1000 ), NetworkCacheSettings( invalidModifiersCacheSize = 10000, @@ -90,11 +91,12 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { rebroadcastCount = 3, minimalFeeAmount = 0, headerChainDiff = 100, - adProofsSuffixLength = 112*1024 + adProofsSuffixLength = 112*1024, + extraIndex = false ) settings.cacheSettings shouldBe CacheSettings( HistoryCacheSettings( - 12, 100, 1000 + 12, 500, 100, 1000 ), NetworkCacheSettings( invalidModifiersCacheSize = 10000, @@ -131,11 +133,12 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { rebroadcastCount = 3, minimalFeeAmount = 0, headerChainDiff = 100, - adProofsSuffixLength = 112*1024 + adProofsSuffixLength = 112*1024, + extraIndex = false ) settings.cacheSettings shouldBe CacheSettings( HistoryCacheSettings( - 12, 100, 1000 + 12, 500, 100, 1000 ), NetworkCacheSettings( invalidModifiersCacheSize = 10000, diff --git a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala index 789a7b06fc..03286d4899 100644 --- a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala +++ b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala @@ -51,7 +51,7 @@ object ChainGenerator extends App with ErgoTestHelpers { val boxSelector: BoxSelector = new ReplaceCompactCollectBoxSelector(30, 2, None) - val startTime = args.headOption.map(_.toLong).getOrElse(timeProvider.time - (blockInterval * 10).toMillis) + val startTime = args.headOption.map(_.toLong).getOrElse(System.currentTimeMillis() - (blockInterval * 10).toMillis) val dir = if (args.length < 2) new File("/tmp/ergo/data") else new File(args(1)) val txsSize: Int = if (args.length < 3) 100 * 1024 else args(2).toInt @@ -62,7 +62,7 @@ object ChainGenerator extends App with ErgoTestHelpers { -1, poPoWBootstrap = false, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 20, - 1000000, 100, adProofsSuffixLength = 112*1024) + 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false) val ms = settings.chainSettings.monetary.copy( minerRewardDelay = RewardDelay ) @@ -76,7 +76,7 @@ object ChainGenerator extends App with ErgoTestHelpers { val votingEpochLength = votingSettings.votingLength val protocolVersion = fullHistorySettings.chainSettings.protocolVersion - val history = ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + val history = ErgoHistory.readOrGenerate(fullHistorySettings)(null) HistoryTestHelpers.allowToApplyOldBlocks(history) val (state, _) = ErgoState.generateGenesisUtxoState(stateDir, StateConstants(fullHistorySettings)) log.info(s"Going to generate a chain at ${dir.getAbsoluteFile} starting from ${history.bestFullBlockOpt}") @@ -93,7 +93,7 @@ object ChainGenerator extends App with ErgoTestHelpers { last: Option[Header], acc: Seq[ModifierId]): Seq[ModifierId] = { val time: Long = last.map(_.timestamp + blockInterval.toMillis).getOrElse(startTime) - if (time < timeProvider.time) { + if (time < System.currentTimeMillis()) { val (txs, lastOut) = genTransactions(last.map(_.height).getOrElse(ErgoHistory.GenesisHeight), initBox, state.stateContext) diff --git a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala index d3c02e37f2..7dabb08224 100644 --- a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala +++ b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala @@ -18,7 +18,6 @@ import org.ergoplatform.wallet.secrets.ExtendedSecretKey import org.ergoplatform.{DataInput, ErgoBox, ErgoScriptPredef} import scorex.core.app.Version import scorex.core.network.PeerSpec -import scorex.core.utils.NetworkTimeProvider import scorex.crypto.authds.ADDigest import scorex.crypto.hash.Digest32 import scorex.util.ScorexLogging @@ -47,7 +46,6 @@ trait ErgoTestConstants extends ScorexLogging { Parameters(0, Parameters.DefaultParameters ++ extension, ErgoValidationSettingsUpdate.empty) } - val timeProvider: NetworkTimeProvider = ErgoTestHelpers.defaultTimeProvider val initSettings: ErgoSettings = ErgoSettings.read(Args(Some("src/test/resources/application.conf"), None)) implicit val settings: ErgoSettings = initSettings diff --git a/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala b/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala index 9888f3583f..bf9d61ac3e 100644 --- a/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala +++ b/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala @@ -1,11 +1,10 @@ package org.ergoplatform.utils import org.ergoplatform.ErgoBoxCandidate -import org.ergoplatform.settings.ErgoSettings import org.ergoplatform.utils.generators.ValidBlocksGenerators import org.scalatest.{EitherValues, OptionValues} import scorex.core.network.peer.PeerInfo -import scorex.core.utils.{NetworkTimeProvider, ScorexEncoding} +import scorex.core.utils.ScorexEncoding import scorex.util.ScorexLogging import java.net.InetSocketAddress @@ -37,8 +36,8 @@ trait ErgoTestHelpers val inetAddr2 = new InetSocketAddress("93.93.93.93", 27017) val peers: Map[InetSocketAddress, PeerInfo] = Map( - inetAddr1 -> PeerInfo(defaultPeerSpec.copy(nodeName = "first"), timeProvider.time()), - inetAddr2 -> PeerInfo(defaultPeerSpec.copy(nodeName = "second"), timeProvider.time()) + inetAddr1 -> PeerInfo(defaultPeerSpec.copy(nodeName = "first"), System.currentTimeMillis()), + inetAddr2 -> PeerInfo(defaultPeerSpec.copy(nodeName = "second"), System.currentTimeMillis()) ) } @@ -46,7 +45,4 @@ object ErgoTestHelpers { implicit val defaultExecutionContext: ExecutionContext = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(10)) - - val defaultTimeProvider: NetworkTimeProvider = - new NetworkTimeProvider(ErgoSettings.read().scorexSettings.ntp) } diff --git a/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala b/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala index 628090c90a..5937979e48 100644 --- a/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala +++ b/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala @@ -49,7 +49,7 @@ trait HistoryTestHelpers extends ErgoPropertyTest { PoPoWBootstrap, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, - rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024 + rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false ) val scorexSettings: ScorexSettings = null val walletSettings: WalletSettings = null @@ -65,7 +65,7 @@ trait HistoryTestHelpers extends ErgoPropertyTest { val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, chainSettings, nodeSettings, scorexSettings, walletSettings, settings.cacheSettings) - ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + ErgoHistory.readOrGenerate(fullHistorySettings)(null) } } diff --git a/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala b/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala index 081a93dde3..d78b852d81 100644 --- a/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala +++ b/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala @@ -96,7 +96,7 @@ trait NodeViewBaseOps extends ErgoTestHelpers { def makeNextBlock(utxoState: UtxoState, txs: Seq[ErgoTransaction]) (implicit ctx: Ctx): ErgoFullBlock = { - val time = timeProvider.time() + val time = System.currentTimeMillis() val parent = getHistory.bestFullBlockOpt validFullBlock(parent, utxoState, txs, Some(time)) } @@ -132,7 +132,7 @@ trait NodeViewTestOps extends NodeViewBaseOps { def getPoolSize(implicit ctx: Ctx): Int = getCurrentView.pool.size - def getRootHash(implicit ctx: Ctx): String = Algos.encode(getCurrentState.rootHash) + def getRootHash(implicit ctx: Ctx): String = Algos.encode(getCurrentState.rootDigest) def getBestFullBlockOpt(implicit ctx: Ctx): Option[ErgoFullBlock] = getHistory.bestFullBlockOpt diff --git a/src/test/scala/org/ergoplatform/utils/Stubs.scala b/src/test/scala/org/ergoplatform/utils/Stubs.scala index d1a1863bf6..66ffffeea1 100644 --- a/src/test/scala/org/ergoplatform/utils/Stubs.scala +++ b/src/test/scala/org/ergoplatform/utils/Stubs.scala @@ -58,7 +58,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with val digestState: DigestState = { boxesHolderGen.map(WrappedUtxoState(_, createTempDir, None, parameters, settings)).map { wus => - DigestState.create(Some(wus.version), Some(wus.rootHash), createTempDir, stateConstants) + DigestState.create(Some(wus.version), Some(wus.rootDigest), createTempDir, stateConstants) } }.sample.value @@ -373,7 +373,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with PoPoWBootstrap, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, internalMinersCount = 1, internalMinerPollingInterval = 1.second,miningPubKeyHex = None, offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, - rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024 + rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false ) val scorexSettings: ScorexSettings = null val walletSettings: WalletSettings = null @@ -383,11 +383,11 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, chainSettings, nodeSettings, scorexSettings, walletSettings, settings.cacheSettings) - ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + ErgoHistory.readOrGenerate(fullHistorySettings)(null) } def syntacticallyValidModifier(history: HT): Header = { - val bestTimestamp = history.bestHeaderOpt.map(_.timestamp + 1).getOrElse(timeProvider.time()) + val bestTimestamp = history.bestHeaderOpt.map(_.timestamp + 1).getOrElse(System.currentTimeMillis()) powScheme.prove( history.bestHeaderOpt, @@ -396,7 +396,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with ADDigest @@ Array.fill(HashLength + 1)(0.toByte), Digest32 @@ Array.fill(HashLength)(0.toByte), Digest32 @@ Array.fill(HashLength)(0.toByte), - Math.max(timeProvider.time(), bestTimestamp), + Math.max(System.currentTimeMillis(), bestTimestamp), Digest32 @@ Array.fill(HashLength)(0.toByte), Array.fill(3)(0: Byte), defaultMinerSecretNumber diff --git a/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala b/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala index 0249379336..5127620226 100644 --- a/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala +++ b/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala @@ -5,9 +5,8 @@ import akka.testkit.TestProbe import org.ergoplatform.mining.emission.EmissionRules import org.ergoplatform.nodeView.ErgoNodeViewRef import org.ergoplatform.settings.{ErgoSettings, Parameters} -import org.ergoplatform.utils.{ErgoTestHelpers, NodeViewTestContext} +import org.ergoplatform.utils.NodeViewTestContext import org.ergoplatform.wallet.utils.TestFileUtils -import scorex.core.utils.NetworkTimeProvider import scala.concurrent.ExecutionContext @@ -22,9 +21,8 @@ class NodeViewFixture(protoSettings: ErgoSettings, parameters: Parameters) exten val nodeViewDir: java.io.File = createTempDir @volatile var settings: ErgoSettings = protoSettings.copy(directory = nodeViewDir.getAbsolutePath) - val timeProvider: NetworkTimeProvider = ErgoTestHelpers.defaultTimeProvider val emission: EmissionRules = new EmissionRules(settings.chainSettings.monetary) - @volatile var nodeViewHolderRef: ActorRef = ErgoNodeViewRef(settings, timeProvider) + @volatile var nodeViewHolderRef: ActorRef = ErgoNodeViewRef(settings) val testProbe = new TestProbe(actorSystem) /** This sender should be imported to make TestProbe work! */ @@ -33,7 +31,7 @@ class NodeViewFixture(protoSettings: ErgoSettings, parameters: Parameters) exten def apply[T](test: self.type => T): T = try test(self) finally stop() def startNodeViewHolder(): Unit = { - nodeViewHolderRef = ErgoNodeViewRef(settings, timeProvider) + nodeViewHolderRef = ErgoNodeViewRef(settings) } def stopNodeViewHolder(): Unit = { diff --git a/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala b/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala index 08bd42a7ca..441bd039f2 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala @@ -108,7 +108,7 @@ trait ChainGenerator extends ErgoTestConstants { EmptyDigest32, EmptyDigest32, tsOpt.getOrElse(prev.map(_.timestamp + control.desiredInterval.toMillis) - .getOrElse(if (useRealTs) timeProvider.time() else 0)), + .getOrElse(if (useRealTs) System.currentTimeMillis() else 0)), extensionHash, Array.fill(3)(0: Byte), defaultMinerSecretNumber @@ -163,7 +163,7 @@ trait ChainGenerator extends ErgoTestConstants { EmptyStateRoot, emptyProofs, txs, - Math.max(timeProvider.time(), prev.map(_.header.timestamp + 1).getOrElse(timeProvider.time())), + Math.max(System.currentTimeMillis(), prev.map(_.header.timestamp + 1).getOrElse(System.currentTimeMillis())), validExtension, Array.fill(3)(0: Byte), defaultMinerSecretNumber diff --git a/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala index 72479d26a3..6edb2f67d0 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala @@ -4,7 +4,6 @@ import org.ergoplatform.ErgoBox.TokenId import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction} -import org.ergoplatform.modifiers.state.UTXOSnapshotChunk import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.state.wrapped.WrappedUtxoState import org.ergoplatform.nodeView.state.{BoxHolder, ErgoStateContext, VotingData} @@ -20,7 +19,7 @@ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.wallet.interpreter.TransactionHintsBag import org.ergoplatform.wallet.utils.Generators import org.ergoplatform.{DataInput, ErgoAddress, ErgoAddressEncoder, ErgoBox, ErgoBoxCandidate, Input, P2PKAddress} -import org.scalacheck.{Arbitrary, Gen} +import org.scalacheck.Gen import scorex.crypto.hash.{Blake2b256, Digest32} import scorex.db.ByteArrayWrapper import scorex.util.encode.Base16 @@ -36,7 +35,7 @@ import scala.util.Random trait ErgoTransactionGenerators extends ErgoGenerators with Generators { - protected implicit val ergoAddressEncoder: ErgoAddressEncoder = + protected implicit val addressEncoder: ErgoAddressEncoder = ErgoAddressEncoder(settings.chainSettings.addressPrefix) val creationHeightGen: Gen[Int] = Gen.choose(0, Int.MaxValue / 2) @@ -292,11 +291,6 @@ trait ErgoTransactionGenerators extends ErgoGenerators with Generators { } yield BlockTransactions(headerId, Header.InitialVersion, txs.foldLeft(Seq.empty[ErgoTransaction])((acc, tx) => if ((acc :+ tx).map(_.size).sum < (Parameters.MaxBlockSizeDefault - 150)) acc :+ tx else acc)) - lazy val randomUTXOSnapshotChunkGen: Gen[UTXOSnapshotChunk] = for { - index: Short <- Arbitrary.arbitrary[Short] - stateElements: Seq[ErgoBox] <- Gen.listOf(ergoBoxGenNoProp) - } yield UTXOSnapshotChunk(stateElements, index) - lazy val invalidErgoFullBlockGen: Gen[ErgoFullBlock] = for { header <- defaultHeaderGen txs <- invalidBlockTransactionsGen diff --git a/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala index a96af11484..31f60d02cb 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala @@ -202,7 +202,7 @@ trait ValidBlocksGenerators val (adProofBytes, updStateDigest) = utxoState.proofsForTransactions(transactions).get - val time = timeOpt.orElse(parentOpt.map(_.header.timestamp + 1)).getOrElse(timeProvider.time()) + val time = timeOpt.orElse(parentOpt.map(_.header.timestamp + 1)).getOrElse(System.currentTimeMillis()) val interlinks = parentOpt.toSeq.flatMap { block => nipopowAlgos.updateInterlinks(block.header, NipopowAlgos.unpackInterlinks(block.extension.fields).get) } @@ -231,7 +231,7 @@ trait ValidBlocksGenerators val (adProofBytes, updStateDigest) = wrappedState.proofsForTransactions(transactions).get - val time = timeOpt.orElse(parentOpt.map(_.timestamp + 1)).getOrElse(timeProvider.time()) + val time = timeOpt.orElse(parentOpt.map(_.timestamp + 1)).getOrElse(System.currentTimeMillis()) val interlinksExtension = nipopowAlgos.interlinksToExtension(nipopowAlgos.updateInterlinks(parentOpt, parentExtensionOpt)) val extension: ExtensionCandidate = parameters.toExtensionCandidate ++ interlinksExtension val votes = Array.fill(3)(0: Byte) diff --git a/src/test/scala/scorex/core/network/DeliveryTrackerSpec.scala b/src/test/scala/scorex/core/network/DeliveryTrackerSpec.scala index a338df74b2..4c7e5ad532 100644 --- a/src/test/scala/scorex/core/network/DeliveryTrackerSpec.scala +++ b/src/test/scala/scorex/core/network/DeliveryTrackerSpec.scala @@ -3,8 +3,8 @@ package scorex.core.network import akka.actor.{ActorRef, Cancellable} import io.circe._ import io.circe.syntax._ +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.utils.ErgoPropertyTest -import scorex.core.ModifierTypeId import scorex.core.network.ModifiersStatus.Received import scorex.testkit.generators.ObjectGenerators import scorex.util.ModifierId @@ -15,7 +15,7 @@ class DeliveryTrackerSpec extends ErgoPropertyTest with ObjectGenerators { forAll(connectedPeerGen(ActorRef.noSender)) { peer => val tracker = DeliveryTracker.empty(settings) val mid: ModifierId = ModifierId @@ "foo" - val mTypeId: ModifierTypeId = ModifierTypeId @@ (104: Byte) + val mTypeId: NetworkObjectTypeId.Value = NetworkObjectTypeId.fromByte(104) tracker.setRequested(mTypeId, mid, peer) { _ => Cancellable.alreadyCancelled} val infoFields = Seq( diff --git a/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala b/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala index e85cef0572..7c9dddf961 100644 --- a/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala +++ b/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala @@ -3,6 +3,7 @@ package scorex.testkit.generators import java.net.{InetAddress, InetSocketAddress, URL} import akka.actor.ActorRef import akka.util.ByteString +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.network.ModePeerFeature import org.ergoplatform.nodeView.state.StateType import org.scalacheck.Gen.{const, some} @@ -11,7 +12,7 @@ import scorex.core.app.Version import scorex.core.network.message.{InvData, ModifiersData} import scorex.core.network._ import scorex.core.network.peer.{PeerInfo, RestApiUrlPeerFeature} -import scorex.core.{ModifierTypeId, NodeViewModifier} +import scorex.core.NodeViewModifier import scorex.util.{ModifierId, bytesToId} trait ObjectGenerators { @@ -49,10 +50,10 @@ trait ObjectGenerators { lazy val modifierIdGen: Gen[ModifierId] = Gen.listOfN(NodeViewModifier.ModifierIdSize, Arbitrary.arbitrary[Byte]) .map(id => bytesToId(id.toArray)) - lazy val modifierTypeIdGen: Gen[ModifierTypeId] = Arbitrary.arbitrary[Byte].map(t => ModifierTypeId @@ t) + lazy val modifierTypeIdGen: Gen[NetworkObjectTypeId.Value] = Arbitrary.arbitrary[Byte].map(t => NetworkObjectTypeId.fromByte(t)) lazy val invDataGen: Gen[InvData] = for { - modifierTypeId: ModifierTypeId <- modifierTypeIdGen + modifierTypeId: NetworkObjectTypeId.Value <- modifierTypeIdGen modifierIds: Seq[ModifierId] <- Gen.nonEmptyListOf(modifierIdGen) if modifierIds.nonEmpty } yield InvData(modifierTypeId, modifierIds) @@ -62,7 +63,7 @@ trait ObjectGenerators { } yield id -> mod lazy val modifiersGen: Gen[ModifiersData] = for { - modifierTypeId: ModifierTypeId <- modifierTypeIdGen + modifierTypeId: NetworkObjectTypeId.Value <- modifierTypeIdGen modifiers: Map[ModifierId, Array[Byte]] <- Gen.nonEmptyMap(modifierWithIdGen).suchThat(_.nonEmpty) } yield ModifiersData(modifierTypeId, modifiers)