From 891bdde9c6cfdd83f09840100e8e12f6f48ff3d8 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 14 Aug 2022 21:58:22 +0200 Subject: [PATCH 01/90] initial commit --- src/main/resources/api/openapi.yaml | 548 ++++++++++++++++++ src/main/resources/application.conf | 3 + src/main/scala/org/ergoplatform/ErgoApp.scala | 13 +- .../org/ergoplatform/http/api/ApiCodecs.scala | 35 +- .../http/api/ExtraIndexApiRoute.scala | 218 +++++++ .../history/HistoryModifierSerializer.scala | 21 + .../nodeView/history/ErgoHistory.scala | 2 + .../nodeView/history/extra/ExtraIndexer.scala | 181 ++++++ .../history/extra/IndexedErgoAddress.scala | 106 ++++ .../history/extra/IndexedErgoBox.scala | 71 +++ .../extra/IndexedErgoTransaction.scala | 87 +++ .../history/extra/IndexedErgoTree.scala | 49 ++ .../settings/NodeConfigurationSettings.scala | 2 + 13 files changed, 1333 insertions(+), 3 deletions(-) create mode 100644 src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala create mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala create mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala create mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala create mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala create mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index e1d74663c3..77878fb186 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -200,6 +200,62 @@ components: example: 1 default: 1 + IndexedErgoBox: + type: object + description: Box indexed with extra information + required: + - box + - confirmationsNum + - address + - creationTransaction + - spendingTransaction + - spendingHeight + - inclusionHeight + - spent + - globalIndex + properties: + box: + $ref: '#/components/schemas/ErgoTransactionOutput' + confirmationsNum: + description: Number of confirmations, if the box is included into the blockchain + type: integer + format: int32 + minimum: 0 + example: 147 + nullable: true + address: + $ref: '#/components/schemas/ErgoAddress' + creationTransaction: + description: Transaction which created the box + $ref: '#/components/schemas/ModifierId' + spendingTransaction: + description: Transaction which created the box + nullable: true + $ref: '#/components/schemas/ModifierId' + spendingHeight: + description: The height the box was spent at + type: integer + format: int32 + minimum: 0 + example: 147 + nullable: true + inclusionHeight: + description: The height the transaction containing the box was included in a block at + type: integer + format: int32 + minimum: 0 + example: 147 + spent: + description: A flag signalling whether the box was spent + type: boolean + example: false + globalIndex: + description: The global index of the box + type: integer + format: int64 + minimum: 0 + example: 83927 + UnsignedErgoTransaction: type: object description: Unsigned Ergo transaction @@ -306,6 +362,70 @@ components: type: integer format: int32 + IndexedErgoTransaction: + type: object + description: Transaction indexed with extra information + required: + - id + - inputs + - dataInputs + - outputs + - inclusionHeight + - numConfirmations + - blockId + - timestamp + - index + - globalIndex + - size + properties: + id: + $ref: '#/components/schemas/TransactionId' + inputs: + description: Transaction inputs + type: array + items: + $ref: '#/components/schemas/ErgoTransactionInput' + dataInputs: + description: Transaction data inputs + type: array + items: + $ref: '#/components/schemas/ErgoTransactionDataInput' + outputs: + description: Transaction outputs + type: array + items: + $ref: '#/components/schemas/ErgoTransactionOutput' + inclusionHeight: + description: Height of a block the transaction was included in + type: integer + format: int32 + example: 20998 + numConfirmations: + description: Number of transaction confirmations + type: integer + format: int32 + example: 20998 + blockId: + description: Id of the block the transaction was included in + allOf: + - $ref: '#/components/schemas/ModifierId' + timestamp: + $ref: '#/components/schemas/Timestamp' + index: + description: index of the transaction in the block it was included in + type: integer + format: int32 + example: 3 + globalIndex: + description: index of the box globally + type: integer + format: int64 + example: 3565445 + size: + description: Size in bytes + type: integer + format: int32 + ErgoAddress: description: Encoded Ergo Address type: string @@ -5064,6 +5184,434 @@ paths: application/json: schema: $ref: '#/components/schemas/EmissionScripts' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/transaction/byId/{txId}: + get: + summary: Retrieve a transaction by its id + operationId: getTxById + tags: + - extra + parameters: + - in: path + name: txId + required: true + description: id of the wanted transaction + schema: + type: string + responses: + '200': + description: transaction with wanted id + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedErgoTransaction' + '404': + description: Transaction with this id doesn't exist + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/transaction/byIndex/{txIndex}: + get: + summary: Retrieve a transaction by global index number + operationId: getTxByIndex + tags: + - extra + parameters: + - in: path + name: txIndex + required: true + description: index of the wanted transaction + schema: + type: number + responses: + '200': + description: transaction with wanted index + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedErgoTransaction' + '404': + description: Transaction with this index doesn't exist + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/transaction/byAddress/{address}: + get: + summary: Retrieve transactions by their associated address + operationId: getTxsByAddress + tags: + - extra + parameters: + - in: path + name: address + required: true + description: adderess associated with transactions + schema: + $ref: '#/components/schemas/ErgoAddress' + - in: query + name: limit + required: false + description: get last n elements, set to negative for all elements + schema: + type: integer + format: int64 + default: 20 + responses: + '200': + description: transactions associated with wanted address + content: + application/json: + schema: + type: array + description: Array of transactions + items: + $ref: '#/components/schemas/IndexedErgoTransaction' + '404': + description: No transactions found for wanted address + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/transaction/range: + get: + summary: Get a range of transaction ids + operationId: getTxRange + tags: + - extra + parameters: + - in: query + name: fromIndex + required: false + description: Min transaction index + schema: + type: integer + format: int64 + default: 0 + - in: query + name: toIndex + required: false + description: Max transaction index + schema: + type: integer + format: int64 + default: 10 + responses: + '200': + description: transactions in wanted range + content: + application/json: + schema: + type: array + description: Array of transactions + items: + $ref: '#/components/schemas/IndexedErgoTransaction' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/box/byId/{boxId}: + get: + summary: Retrieve a box by its id + operationId: getBoxById + tags: + - extra + parameters: + - in: path + name: boxId + required: true + description: id of the wanted box + schema: + type: string + responses: + '200': + description: box with wanted id + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedErgoBox' + '404': + description: No box found with wanted id + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/box/byIndex/{boxIndex}: + get: + summary: Retrieve a box by global index number + operationId: getBoxByIndex + tags: + - extra + parameters: + - in: path + name: boxIndex + required: true + description: index of the wanted box + schema: + type: number + responses: + '200': + description: box with wanted index + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedErgoBox' + '404': + description: Box with this index doesn't exist + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/box/byAddress/{address}: + get: + summary: Retrieve boxes by their associated address + operationId: getBoxesByAddress + tags: + - extra + parameters: + - in: path + name: address + required: true + description: adderess associated with boxes + schema: + $ref: '#/components/schemas/ErgoAddress' + - in: query + name: limit + required: false + description: get last n elements, set to negative for all elements + schema: + type: integer + format: int64 + default: 20 + responses: + '200': + description: boxes associated with wanted address + content: + application/json: + schema: + type: array + description: Array of boxes + items: + $ref: '#/components/schemas/IndexedErgoTransaction' + '404': + description: No boxes found for wanted address + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/box/unspent/byAddress/{address}: + get: + summary: Retrieve unspent boxes by their associated address + operationId: getBoxesByAddressUnspent + tags: + - extra + parameters: + - in: path + name: address + required: true + description: adderess associated with unspent boxes + schema: + $ref: '#/components/schemas/ErgoAddress' + - in: query + name: limit + required: false + description: get last n elements, set to negative for all elements + schema: + type: integer + format: int64 + default: 20 + responses: + '200': + description: unspent boxes associated with wanted address + content: + application/json: + schema: + type: array + description: Array of boxes + items: + $ref: '#/components/schemas/IndexedErgoTransaction' + '404': + description: No unspent boxes found for wanted address + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/box/range: + get: + summary: Get a range of box ids + operationId: getBoxRange + tags: + - extra + parameters: + - in: query + name: fromIndex + required: false + description: Min box index + schema: + type: integer + format: int64 + default: 0 + - in: query + name: toIndex + required: false + description: Max box index + schema: + type: integer + format: int64 + default: 10 + responses: + '200': + description: boxes in wanted range + content: + application/json: + schema: + type: array + description: Array of boxes + items: + $ref: '#/components/schemas/IndexedErgoBox' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/box/byErgoTree/{ergoTreeHex}: + get: + summary: Retrieve boxes by their associated ergotree + operationId: getBoxesByErgoTree + tags: + - extra + parameters: + - in: path + name: ergoTreeHex + required: true + description: hex encoded ergotree + schema: + type: string + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' + - in: query + name: limit + required: false + description: get last n elements, set to negative for all elements + schema: + type: integer + format: int64 + default: 20 + responses: + '200': + description: boxes with wanted ergotree + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedErgoBox' + '404': + description: No box found with wanted ergotree + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /extra/box/unspent/byErgoTree/{ergoTreeHex}: + get: + summary: Retrieve unspent boxes by their associated ergotree + operationId: getBoxesByErgoTreeUnspent + tags: + - extra + parameters: + - in: path + name: ergoTreeHex + required: true + description: hex encoded ergotree + schema: + type: string + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' + - in: query + name: limit + required: false + description: get last n elements, set to negative for all elements + schema: + type: integer + format: int64 + default: 20 + responses: + '200': + description: unspent boxes with wanted ergotree + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedErgoBox' + '404': + description: No unspent box found with wanted ergotree + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' default: description: Error content: diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index c6f1f499cb..eb2f953a81 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -70,6 +70,9 @@ ergo { # Minimal fee amount of transactions mempool accepts minimalFeeAmount = 1000000 + # If true, the node will store all transactions, boxes and addresses. + extraIndex = false + # List with hex-encoded identifiers of transactions banned from getting into memory pool blacklistedTransactions = [] diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index a2d3c5b798..9b5eb0261f 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -13,6 +13,7 @@ import org.ergoplatform.mining.ErgoMiner import org.ergoplatform.mining.ErgoMiner.StartMining import org.ergoplatform.network.{ErgoNodeViewSynchronizer, ErgoSyncTracker, ModePeerFeature} import org.ergoplatform.nodeView.history.ErgoSyncInfoMessageSpec +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef import org.ergoplatform.nodeView.{ErgoNodeViewRef, ErgoReadersHolderRef} import org.slf4j.{Logger, LoggerFactory} import org.ergoplatform.settings.{Args, ErgoSettings, NetworkType} @@ -28,7 +29,7 @@ import scorex.util.ScorexLogging import java.net.InetSocketAddress import scala.concurrent.{ExecutionContext, Future} -import scala.io.Source +import scala.io.{Codec, Source} class ErgoApp(args: Args) extends ScorexLogging { @@ -100,6 +101,13 @@ class ErgoApp(args: Args) extends ScorexLogging { None } + // Create an instance of ExtraIndexer actor if "extraIndex = true" in config + private val indexerRefOpt: Option[ActorRef] = + if(ergoSettings.nodeSettings.extraIndex) + Some(ExtraIndexerRef(ergoSettings.chainSettings)) + else + None + ExtraIndexerRef.setAddressEncoder(ergoSettings.addressEncoder) // initialize an accessible address encoder regardless of extra indexing being enabled private val syncTracker = ErgoSyncTracker(scorexSettings.network, timeProvider) private val statsCollectorRef: ActorRef = ErgoStatsCollectorRef(readersHolderRef, networkControllerRef, syncTracker, ergoSettings, timeProvider) @@ -124,6 +132,7 @@ class ErgoApp(args: Args) extends ScorexLogging { private val apiRoutes: Seq[ApiRoute] = Seq( EmissionApiRoute(ergoSettings), ErgoUtilsApiRoute(ergoSettings), + ExtraIndexApiRoute(readersHolderRef, ergoSettings, indexerRefOpt), ErgoPeersApiRoute(peerManagerRef, networkControllerRef, syncTracker, deliveryTracker, scorexSettings.restApi), InfoApiRoute(statsCollectorRef, scorexSettings.restApi, timeProvider), BlocksApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings), @@ -176,7 +185,7 @@ class ErgoApp(args: Args) extends ScorexLogging { root.setLevel(Level.toLevel(ergoSettings.scorexSettings.logging.level)) } - private def swaggerConfig: String = Source.fromResource("api/openapi.yaml").getLines.mkString("\n") + private def swaggerConfig: String = Source.fromResource("api/openapi.yaml")(Codec.UTF8).getLines.mkString("\n") private def run(): Future[ServerBinding] = { require(scorexSettings.network.agentName.length <= ErgoApp.ApplicationNameLimit) diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index 94979e5228..06419df406 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -1,7 +1,6 @@ package org.ergoplatform.http.api import java.math.BigInteger - import io.circe._ import org.bouncycastle.util.BigIntegers import org.ergoplatform.{ErgoBox, ErgoLikeContext, ErgoLikeTransaction, JsonCodecs, UnsignedErgoLikeTransaction} @@ -29,6 +28,7 @@ import sigmastate.interpreter._ import sigmastate.interpreter.CryptoConstants.EcPointType import io.circe.syntax._ import org.ergoplatform.http.api.requests.{CryptoResult, ExecuteRequest, HintExtractionRequest} +import org.ergoplatform.nodeView.history.extra.{ExtraIndexerRef, IndexedErgoAddress, IndexedErgoBox, IndexedErgoTransaction} import org.ergoplatform.wallet.interface4j.SecretString import scorex.crypto.authds.{LeafData, Side} import scorex.crypto.authds.merkle.MerkleProof @@ -469,6 +469,39 @@ trait ApiCodecs extends JsonCodecs { fields.asJson } + implicit val indexedBoxEncoder: Encoder[IndexedErgoBox] = { iEb => + iEb.box.asJson.deepMerge(Json.obj( + "globalIndex" -> iEb.globalIndex.asJson, + "inclusionHeight" -> iEb.inclusionHeightOpt.asJson, + "address" -> ExtraIndexerRef.getAddressEncoder.toString(iEb.getAddress).asJson, + "spentTransactionId" -> iEb.spendingTxIdOpt.asJson + )) + } + + implicit val indexedTxEncoder: Encoder[IndexedErgoTransaction] = { tx => + Json.obj( + "id" -> tx.id.asJson, + "blockId" -> tx.blockId.asJson, + "inclusionHeight" -> tx.inclusionHeight.asJson, + "timestamp" -> tx.timestamp.asJson, + "index" -> tx.index.asJson, + "globalIndex" -> tx.globalIndex.asJson, + "numConfirmations" -> tx.numConfirmations.asJson, + "inputs" -> tx.inputs.asJson, + "dataInputs" -> tx.dataInputs.asJson, + "outputs" -> tx.outputs.asJson, + "size" -> tx.txSize.asJson) + } + + implicit val indexedAddressEncoder: Encoder[IndexedErgoAddress] = { address => + Json.obj( + "transactions" -> address.transactions(-1).map(_.asJson).asJson, + "txCount" -> address.transactions(-1).size.asJson, + "utxos" -> address.utxos(-1).asJson, + "balance" -> address.utxos(-1).map(_.box.value).sum.asJson, + "tokens" -> address.utxos(-1).map(_.box.additionalTokens.toArray.toSeq.asJson).asJson + ) + } } trait ApiEncoderOption diff --git a/src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala new file mode 100644 index 0000000000..924356e5e8 --- /dev/null +++ b/src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala @@ -0,0 +1,218 @@ +package org.ergoplatform.http.api + +import akka.actor.{ActorRef, ActorRefFactory} +import akka.http.scaladsl.server.{Directive, Route} +import akka.pattern.ask +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} +import org.ergoplatform.nodeView.ErgoReadersHolder.GetDataFromHistory +import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{box_indexNumHash, ergoTreeHash, tx_indexNumHash} +import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoAddressSerializer, IndexedErgoBox, IndexedErgoTransaction, IndexedErgoTree} +import org.ergoplatform.settings.ErgoSettings +import scorex.core.api.http.ApiError.BadRequest +import scorex.core.api.http.ApiResponse +import scorex.core.settings.RESTApiSettings +import scorex.util.encode.Base16 +import scorex.util.{ModifierId, bytesToId} +import sigmastate.Values.ErgoTree +import sigmastate.serialization.ErgoTreeSerializer + +import scala.concurrent.Future +import scala.util.{Failure, Success} + +case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings, extraIndexerOpt: Option[ActorRef]) + (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute with ApiCodecs { + + val settings: RESTApiSettings = ergoSettings.scorexSettings.restApi + + val paging: Directive[(Long, Long)] = parameters("fromIndex".as[Long] ? 0L, "toIndex".as[Long] ? 10L) + + val lastN: Directive[Tuple1[Long]] = parameter("limit".as[Long] ? 20L) + + val addressEncoder: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder + + private val MaxTxs = 16384 + private val MaxBoxes = MaxTxs * 3 + + override val route: Route = pathPrefix("extra") { + getTxByIdR ~ + getTxByIndexR ~ + getTxsByAddressR ~ + getTxRangeR ~ + getBoxByIdR ~ + getBoxByIndexR ~ + getBoxesByAddressR ~ + getBoxesByAddressUnspentR ~ + getBoxRangeR ~ + getBoxesByErgoTreeR ~ + getBoxesByErgoTreeUnspentR + } + + private def getHistory: Future[ErgoHistoryReader] = + (readersHolder ? GetDataFromHistory[ErgoHistoryReader](r => r)).mapTo[ErgoHistoryReader] + + private def getAddress(addr: ErgoAddress)(implicit history: ErgoHistoryReader): Option[IndexedErgoAddress] = { + val x: Option[IndexedErgoAddress] = history.typedModifierById[IndexedErgoAddress](IndexedErgoAddressSerializer.addressToModifierId(addr)) + if(x.isDefined) log.info(s"Address: ${addressEncoder.toString(x.get.address)} => ${x.get.txIds}, ${x.get.boxIds}") + x + } + + private def getTxById(id: ModifierId)(implicit history: ErgoHistoryReader): Option[IndexedErgoTransaction] = + history.typedModifierById[IndexedErgoTransaction](id) match { + case Some(tx) => Some(tx.retrieveBody(history)) + case None => None + } + + + private def getTxByIdF(id: ModifierId) : Future[Option[IndexedErgoTransaction]] = + getHistory.map { history => + getTxById(id)(history) + } + + private def getTxByIdR: Route = (get & pathPrefix("transaction" / "byId") & modifierId) { id => + ApiResponse(getTxByIdF(bytesToId(Base16.decode(id).get))) + } + + private def getTxByIndex(index: Long): Future[Option[IndexedErgoTransaction]] = + getHistory.map { history => + getTxById(bytesToId(history.modifierBytesById(bytesToId(tx_indexNumHash(index))).get))(history) // prepend 0 byte to circumvent "removing modifier type byte with .tail" + } + + private def getTxByIndexR: Route = (pathPrefix("transaction" / "byIndex" / LongNumber) & get) { index => + ApiResponse(getTxByIndex(index)) + } + + private def getTxsByAddress(addr: ErgoAddress, lastN: Long): Future[Option[Seq[IndexedErgoTransaction]]] = + getHistory.map { history => + getAddress(addr)(history) match { + case Some(addr) => Some(addr.retrieveBody(history).transactions(lastN)) + case None => None + } + } + + private def getTxsByAddressR: Route = (get & pathPrefix("transaction" / "byAddress") & path(Segment) & lastN) { (address, limit) => + addressEncoder.fromString(address) match { + case Success(addr) => ApiResponse(getTxsByAddress(addr, limit)) + case Failure(_) => BadRequest("Incorrect address format") + } + } + + private def getTxRange(fromHeight: Long, toHeight: Long): Future[Seq[ModifierId]] = + getHistory.map { history => + (for(n <- fromHeight to toHeight) + yield bytesToId(history.modifierBytesById(bytesToId(tx_indexNumHash(n))).get) + ).toSeq + } + + private def getTxRangeR: Route = (pathPrefix("transaction" / "range") & paging) { (fromIndex, toIndex) => + if (toIndex < fromIndex) { + BadRequest("toIndex < fromIndex") + } else if (fromIndex - toIndex > MaxTxs) { + BadRequest(s"No more than $MaxTxs transactions can be requested") + } else { + ApiResponse(getTxRange(fromIndex, toIndex)) + } + } + + private def getBoxById(id: ModifierId)(implicit history: ErgoHistoryReader): Option[IndexedErgoBox] = + history.typedModifierById[IndexedErgoBox](id) + + private def getBoxByIdF(id: ModifierId): Future[Option[IndexedErgoBox]] = + getHistory.map { history => + getBoxById(id)(history) + } + + private def getBoxByIdR: Route = (get & pathPrefix("box" / "byId") & modifierId) { id => + ApiResponse(getBoxByIdF(bytesToId(Base16.decode(id).get))) + } + + private def getBoxByIndex(index: Long): Future[Option[IndexedErgoBox]] = + getHistory.map { history => + getBoxById(bytesToId(history.modifierBytesById(bytesToId(box_indexNumHash(index))).get))(history) + } + + private def getBoxByIndexR: Route = (pathPrefix("box" / "byIndex" / LongNumber) & get) { index => + ApiResponse(getBoxByIndex(index)) + } + + private def getBoxesByAddress(addr: ErgoAddress, lastN: Long): Future[Seq[IndexedErgoBox]] = + getHistory.map { history => + getAddress(addr)(history) match { + case Some(addr) => addr.retrieveBody(history).boxes(lastN) + case None => Seq.empty[IndexedErgoBox] + } + } + + private def getBoxesByAddressR: Route = (get & pathPrefix("box" / "byAddress") & path(Segment) & lastN) { (address, limit) => + addressEncoder.fromString(address) match { + case Success(addr) => ApiResponse(getBoxesByAddress(addr, limit)) + case Failure(_) => BadRequest("Incorrect address format") + } + } + + private def getBoxesByAddressUnspent(addr: ErgoAddress, lastN: Long): Future[Seq[IndexedErgoBox]] = + getHistory.map { history => + getAddress(addr)(history) match { + case Some(addr) => addr.retrieveBody(history).utxos(lastN) + case None => Seq.empty[IndexedErgoBox] + } + } + + private def getBoxesByAddressUnspentR: Route = (get & pathPrefix("transaction" / "unspent" / "byAddress") & path(Segment) & lastN) { (address, limit) => + addressEncoder.fromString(address) match { + case Success(addr) => ApiResponse(getBoxesByAddressUnspent(addr, limit)) + case Failure(_) => BadRequest("Incorrect address format") + } + } + + private def getBoxRange(fromHeight: Long, toHeight: Long): Future[Seq[ModifierId]] = + getHistory.map { history => + (for(n <- fromHeight to toHeight) + yield bytesToId(history.modifierBytesById(bytesToId(box_indexNumHash(n))).get) + ).toSeq + } + + private def getBoxRangeR: Route = (pathPrefix("box" / "range") & paging) { (fromIndex, toIndex) => + if (toIndex < fromIndex) { + BadRequest("toIndex < fromIndex") + } else if (fromIndex - toIndex > MaxBoxes) { + BadRequest(s"No more than $MaxBoxes boxes can be requested") + } else { + ApiResponse(getBoxRange(fromIndex, toIndex)) + } + } + + private def getBoxesByErgoTree(tree: ErgoTree, lastN: Long): Future[Seq[IndexedErgoBox]] = + getHistory.map { history => + history.typedModifierById[IndexedErgoTree](bytesToId(ergoTreeHash(tree))) match { + case Some(iEt) => iEt.retrieveBody(history, lastN) + case None => Seq.empty[IndexedErgoBox] + } + } + + private def getBoxesByErgoTreeR: Route = (get & pathPrefix("box" / "byErgoTree") & path(Segment) & lastN) { (tree, limit) => + try { + ApiResponse(getBoxesByErgoTree(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(Base16.decode(tree).get), limit)) + }catch { + case e: Exception => BadRequest(s"${e.getMessage}") + } + } + + private def getBoxesByErgoTreeUnspent(tree: ErgoTree, lastN: Long): Future[Seq[IndexedErgoBox]] = + getHistory.map { history => + history.typedModifierById[IndexedErgoTree](bytesToId(ergoTreeHash(tree))) match { + case Some(iEt) => iEt.retrieveBody(history, lastN).filter(!_.trackedBox.isSpent) + case None => Seq.empty[IndexedErgoBox] + } + } + + private def getBoxesByErgoTreeUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byErgoTree") & path(Segment) & lastN) { (tree, limit) => + try { + ApiResponse(getBoxesByErgoTreeUnspent(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(Base16.decode(tree).get), limit)) + }catch { + case e: Exception => BadRequest(s"${e.getMessage}") + } + } + +} + diff --git a/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala b/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala index d1ef4eb208..ad29660c1b 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala @@ -3,6 +3,7 @@ package org.ergoplatform.modifiers.history import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionSerializer} import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} +import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoAddressSerializer, IndexedErgoBox, IndexedErgoBoxSerializer, IndexedErgoTransaction, IndexedErgoTransactionSerializer, IndexedErgoTree, IndexedErgoTreeSerializer} import scorex.core.serialization.ScorexSerializer import scorex.util.serialization.{Reader, Writer} @@ -22,6 +23,18 @@ object HistoryModifierSerializer extends ScorexSerializer[BlockSection] { case m: Extension => w.put(Extension.modifierTypeId) ExtensionSerializer.serialize(m, w) + case m: IndexedErgoTree => + w.put(IndexedErgoTree.modifierTypeId) + IndexedErgoTreeSerializer.serialize(m, w) + case m: IndexedErgoAddress => + w.put(IndexedErgoAddress.modifierTypeId) + IndexedErgoAddressSerializer.serialize(m, w) + case m: IndexedErgoTransaction => + w.put(IndexedErgoTransaction.modifierTypeId) + IndexedErgoTransactionSerializer.serialize(m, w) + case m: IndexedErgoBox => + w.put(IndexedErgoBox.modifierTypeId) + IndexedErgoBoxSerializer.serialize(m, w) case m => throw new Error(s"Serialization for unknown modifier: $m") } @@ -37,6 +50,14 @@ object HistoryModifierSerializer extends ScorexSerializer[BlockSection] { BlockTransactionsSerializer.parse(r) case Extension.`modifierTypeId` => ExtensionSerializer.parse(r) + case IndexedErgoTree.`modifierTypeId` => + IndexedErgoTreeSerializer.parse(r) + case IndexedErgoAddress.`modifierTypeId` => + IndexedErgoAddressSerializer.parse(r) + case IndexedErgoTransaction.`modifierTypeId` => + IndexedErgoTransactionSerializer.parse(r) + case IndexedErgoBox.`modifierTypeId` => + IndexedErgoBoxSerializer.parse(r) case m => throw new Error(s"Deserialization for unknown type byte: $m") } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala index d2ff73083f..c23df29c40 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala @@ -8,6 +8,7 @@ import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.header.{Header, PreGenesisHeader} import org.ergoplatform.modifiers.state.UTXOSnapshotChunk import org.ergoplatform.modifiers.{NonHeaderBlockSection, ErgoFullBlock, BlockSection} +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRefHolder import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.nodeView.history.storage.modifierprocessors._ import org.ergoplatform.nodeView.history.storage.modifierprocessors.popow.{EmptyPoPoWProofsProcessor, FullPoPoWProofsProcessor} @@ -320,6 +321,7 @@ object ErgoHistory extends ScorexLogging { repairIfNeeded(history) log.info("History database read") + ExtraIndexerRefHolder.start(history) //start extra indexer, if enabled history } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala new file mode 100644 index 0000000000..5871b4b73a --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -0,0 +1,181 @@ +package org.ergoplatform.nodeView.history.extra + +import akka.actor.{Actor, ActorRef, ActorSystem, Props} +import org.ergoplatform.ErgoBox.BoxId +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} +import org.ergoplatform.modifiers.history.BlockTransactions +import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier +import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{box_indexNumHash, ergoTreeHash, tx_indexNumHash} +import org.ergoplatform.nodeView.history.storage.HistoryStorage +import org.ergoplatform.settings.{Algos, ChainSettings, ErgoAlgos} +import scorex.db.ByteArrayWrapper +import scorex.util.{ScorexLogging, bytesToId} +import sigmastate.Values.ErgoTree + +import java.nio.ByteBuffer +import scala.collection.mutable.ArrayBuffer + +class ExtraIndex(chainSettings: ChainSettings) + extends Actor with ScorexLogging { + + private val IndexedHeightKey: ByteArrayWrapper = ByteArrayWrapper.apply(Algos.hash("indexed height")) + private var indexedHeight: Int = 0 + + private val GlobalTxIndexKey: ByteArrayWrapper = ByteArrayWrapper.apply(Algos.hash("txns height")) + private var globalTxIndex: Long = 0L + + private val GlobalBoxIndexKey: ByteArrayWrapper = ByteArrayWrapper.apply(Algos.hash("boxes height")) + private var globalBoxIndex: Long = 0L + + private var done: Boolean = false + + private def chainHeight: Int = _history.fullBlockHeight + + private var _history: ErgoHistory = null + private def history: ErgoHistoryReader = _history.asInstanceOf[ErgoHistoryReader] + private def historyStorage: HistoryStorage = _history.historyStorage + + private def index(bt: BlockTransactions, height: Int) = { + + var txIndexTip: IndexedErgoTransaction = null + var boxIndexTip: IndexedErgoBox = null + + //process transactions + bt.txIds.indices.foreach(n => { + val tx = IndexedErgoTransaction(bytesToId(bt.txIds(n)), height, globalTxIndex) + historyStorage.insert(bt.txIds(n), tx.toBytes) // tx by id + historyStorage.insert(tx_indexNumHash(globalTxIndex), 0.toByte +: bt.txIds(n)) // tx id by global tx number (repend 0 byte to circumvent "removing modifier type byte with .tail") + globalTxIndex += 1 + txIndexTip = tx + }) + + bt.txs.foreach(tx => { + + //process tx inputs + if(height != 1) { //only after 1st block (skip genesis box) + tx.inputs.indices.foreach(n => { + val id: BoxId = tx.inputs(n).boxId + history.typedModifierById[IndexedErgoBox](bytesToId(id)) match { + case Some(iEb) => + boxIndexTip = iEb.asSpent(tx.id, height) + historyStorage.insert(id, boxIndexTip.toBytes) // box by id + case None => log.warn(s"Input for box ${ErgoAlgos.encode(id)} not found in database (this shouldn't happen)") + } + }) + } + + //process tx outputs + var outputs: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] + tx.outputs.indices.foreach(n => { + boxIndexTip = new IndexedErgoBox(Some(height), None, None, tx.outputs(n), globalBoxIndex, Some(chainHeight - height)) + historyStorage.insert(boxIndexTip.serializedId, boxIndexTip.toBytes) // box by id + historyStorage.insert(box_indexNumHash(globalBoxIndex), 0.toByte +: boxIndexTip.serializedId) // box id by global box number (repend 0 byte to circumvent "removing modifier type byte with .tail") + val iEt: IndexedErgoTree = history.typedModifierById[IndexedErgoTree](bytesToId(ergoTreeHash(boxIndexTip.box.ergoTree))) match { + case Some(x) => IndexedErgoTree(x.treeHash, x.boxIds :+ bytesToId(boxIndexTip.box.id)) // ergotree found, update + case None => IndexedErgoTree(bytesToId(ergoTreeHash(boxIndexTip.box.ergoTree)), Seq(bytesToId(boxIndexTip.box.id))) // ergotree not found, record + } + historyStorage.insert(iEt.serializedId, iEt.getBytes) // box id by ergotree + outputs :+= boxIndexTip + globalBoxIndex += 1 + }) + + //process boxes by address + val boxesGroupedByAddress : Array[(ErgoAddress, Seq[BoxId])] = outputs.map(x => (x.getAddress, x.box.id)).groupBy(_._1).mapValues(_.map(_._2).toSeq).toArray + for(addressWithBoxes <- boxesGroupedByAddress) { + val addr: IndexedErgoAddress = + history.typedModifierById[IndexedErgoAddress](IndexedErgoAddressSerializer.addressToModifierId(addressWithBoxes._1)) match { + case Some(iEa) => new IndexedErgoAddress(addressWithBoxes._1, iEa.txIds :+ tx.id, iEa.boxIds ++ addressWithBoxes._2) //address found, update + case None => new IndexedErgoAddress(addressWithBoxes._1, Seq(tx.id), addressWithBoxes._2) //address not found, record + } + historyStorage.insert(addr.serializedId, addr.getBytes) + } + + }) + + log.info(s"Indexed block #$height [transactions: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] - progress: $height / $chainHeight") + if(done) indexedHeight = height // after the indexer caught up with the chain height gets updated here + writeProgress() + } + + private val empty: Seq[BlockSection] = Seq.empty[BlockSection] + private def writeProgress() = { + historyStorage.insert(Seq((IndexedHeightKey , ByteBuffer.allocate(4).putInt (indexedHeight ).array), + (GlobalTxIndexKey , ByteBuffer.allocate(8).putLong(globalTxIndex ).array), + (GlobalBoxIndexKey, ByteBuffer.allocate(8).putLong(globalBoxIndex).array)), empty) + } + + private def run() = { + + indexedHeight = ByteBuffer.wrap(historyStorage.getIndex(IndexedHeightKey ).getOrElse(Array.fill[Byte](4){0})).getInt + globalTxIndex = ByteBuffer.wrap(historyStorage.getIndex(GlobalTxIndexKey ).getOrElse(Array.fill[Byte](8){0})).getLong + globalBoxIndex = ByteBuffer.wrap(historyStorage.getIndex(GlobalBoxIndexKey).getOrElse(Array.fill[Byte](8){0})).getLong + + log.info(s"Started extra indexer at height $indexedHeight") + + while(indexedHeight < chainHeight) { + indexedHeight += 1 + index(history.bestBlockTransactionsAt(indexedHeight).get, indexedHeight) + } + + done = true + + log.info("Indexer caught up with chain") + } + + override def preStart(): Unit = { + context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) + } + + override def postStop(): Unit = { + if(indexedHeight != 0) // only after done or stopped, not at shutdown + log.info(s"Stopped extra indexer at height ${indexedHeight - 1}") + } + + override def receive: Receive = { + case SemanticallySuccessfulModifier(fb: ErgoFullBlock) => if(done) index(fb.blockTransactions, fb.height) // after the indexer caught up with the chain, stay up to date + case Start(history: ErgoHistory) => + _history = history + run() + } +} + +object ExtraIndexerRef { + + object ReceivableMessages { + case class Start(history: ErgoHistory) + } + + private var _ae: ErgoAddressEncoder = _ + + def getAddressEncoder: ErgoAddressEncoder = _ae + + def setAddressEncoder(encoder: ErgoAddressEncoder): Unit = if(_ae == null) _ae = encoder + + def apply(chainSettings: ChainSettings)(implicit system: ActorSystem): ActorRef = { + val actor = system.actorOf(Props.create(classOf[ExtraIndex], chainSettings)) + _ae = chainSettings.addressEncoder + ExtraIndexerRefHolder.init(actor) + actor + } + + def tx_indexNumHash(index: Long): Array[Byte] = Algos.hash("txns height " + index) + def box_indexNumHash(index: Long): Array[Byte] = Algos.hash("boxes height " + index) + + def ergoTreeHash(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) +} + +object ExtraIndexerRefHolder { + + private var _actor: Option[ActorRef] = None + private var running: Boolean = false + + def start(history: ErgoHistory) = if(_actor.isDefined && !running) { + _actor.get ! Start(history) + running = true + } + + protected[extra] def init(actor: ActorRef) = _actor = Some(actor) +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala new file mode 100644 index 0000000000..8ba32f827b --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -0,0 +1,106 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, P2PKAddress, Pay2SAddress, Pay2SHAddress} +import org.ergoplatform.ErgoAddressEncoder.{ChecksumLength, hash256} +import org.ergoplatform.ErgoBox.BoxId +import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.settings.Algos +import scorex.core.ModifierTypeId +import scorex.core.serialization.ScorexSerializer +import scorex.crypto.authds.ADKey +import scorex.util.{ByteArrayOps, ModifierId, bytesToId} +import scorex.util.serialization.{Reader, Writer} +import sigmastate.basics.DLogProtocol.ProveDlog +import sigmastate.serialization.{ErgoTreeSerializer, GroupElementSerializer, SigmaSerializer} + +class IndexedErgoAddress(val address: ErgoAddress, + val txIds: Seq[ModifierId], + val boxIds: Seq[BoxId]) extends BlockSection { + + override val sizeOpt: Option[Int] = None + override def serializedId: Array[Byte] = Algos.hash(IndexedErgoAddressSerializer.addressToBytes(address)) + override def parentId: ModifierId = null + override val modifierTypeId: ModifierTypeId = IndexedErgoAddress.modifierTypeId + override type M = IndexedErgoAddress + override def serializer: ScorexSerializer[IndexedErgoAddress] = IndexedErgoAddressSerializer + + def getBytes: Array[Byte] = serializer.toBytes(this) + + private var _transactions: Seq[IndexedErgoTransaction] = Seq.empty[IndexedErgoTransaction] + private var _boxes: Seq[IndexedErgoBox] = Seq.empty[IndexedErgoBox] + private var _utxos: Seq[IndexedErgoBox] = _ + + def transactions(lastN: Long = 20L): Seq[IndexedErgoTransaction] = + if(lastN > 0) + _transactions.slice(math.max((_transactions.size - lastN).toInt, 0), _transactions.size) + else + _transactions + + def boxes(lastN: Long = 20L): Seq[IndexedErgoBox] = + if(lastN > 0) + _boxes.slice(math.max((_boxes.size - lastN).toInt, 0), _boxes.size) + else + _boxes + + def utxos(lastN: Long = 20L): Seq[IndexedErgoBox] = { + if(lastN > 0) + _utxos.slice(math.max((_utxos.size - lastN).toInt, 0), _utxos.size) + else + _utxos + } + + def retrieveBody(history: ErgoHistoryReader): IndexedErgoAddress = { + _transactions = txIds.map(history.typedModifierById[IndexedErgoTransaction](_).get.retrieveBody(history)) + _boxes = boxIds.map(x => history.typedModifierById[IndexedErgoBox](bytesToId(x)).get) + _utxos = _boxes.filter(!_.trackedBox.isSpent) + this + } +} + +object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] { + + def addressToBytes(address: ErgoAddress): Array[Byte] = { + val withNetworkByte = (ExtraIndexerRef.getAddressEncoder.networkPrefix + address.addressTypePrefix).toByte +: address.contentBytes + withNetworkByte ++ hash256(withNetworkByte).take(ChecksumLength) + } + + def addressToModifierId(address: ErgoAddress): ModifierId = bytesToId(Algos.hash(addressToBytes(address))) + + def bytesToAddress(bytes: Array[Byte]): ErgoAddress = { + val contentBytes: Array[Byte] = bytes.slice(1, bytes.length - ChecksumLength) + implicit val encoder: ErgoAddressEncoder = ExtraIndexerRef.getAddressEncoder + (bytes(0) & 0x0F).toByte match { + case P2PKAddress. addressTypePrefix => new P2PKAddress(ProveDlog(GroupElementSerializer.parseTry(SigmaSerializer.startReader(contentBytes)).get), contentBytes) + case Pay2SHAddress.addressTypePrefix => new Pay2SHAddress(contentBytes) + case Pay2SAddress. addressTypePrefix => new Pay2SAddress(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(contentBytes), contentBytes) + } + } + + override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = { + w.putUByte(IndexedErgoAddress.modifierTypeId) + val addressBytes: Array[Byte] = addressToBytes(iEa.address) + w.putUShort(addressBytes.length) + w.putBytes(addressBytes) + w.putUInt(iEa.txIds.length) + iEa.txIds.foreach(m => w.putBytes(m.toBytes)) + w.putUInt(iEa.boxIds.length) + iEa.boxIds.foreach(w.putBytes(_)) + } + + override def parse(r: Reader): IndexedErgoAddress = { + val addressLen: Int = r.getUShort() + val address: ErgoAddress = bytesToAddress(r.getBytes(addressLen)) + val txnsLen: Long = r.getUInt() + var txns: Seq[ModifierId] = Seq.empty[ModifierId] + for(n <- 1L to txnsLen) txns = txns :+ r.getBytes(32).toModifierId + val boxesLen: Long = r.getUInt() + var boxes: Seq[BoxId] = Seq.empty[BoxId] + for(n <- 1L to boxesLen) boxes = boxes :+ ADKey @@ r.getBytes(32) + new IndexedErgoAddress(address, txns, boxes) + } +} + +object IndexedErgoAddress { + val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 15.toByte +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala new file mode 100644 index 0000000000..70af3f839a --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -0,0 +1,71 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.{ErgoAddress, ErgoBox, Pay2SAddress} +import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.nodeView.wallet.WalletBox +import org.ergoplatform.wallet.Constants.ScanId +import org.ergoplatform.wallet.boxes.{ErgoBoxSerializer, TrackedBox} +import scorex.core.ModifierTypeId +import scorex.core.serialization.ScorexSerializer +import scorex.util.{ModifierId, bytesToId, idToBytes} +import scorex.util.serialization.{Reader, Writer} + +class IndexedErgoBox(val inclusionHeightOpt: Option[Int], + val spendingTxIdOpt: Option[ModifierId], + val spendingHeightOpt: Option[Int], + val box: ErgoBox, + val globalIndex: Long, + val confirmations: Option[Int]) + extends WalletBox(new TrackedBox(box.transactionId, + box.index, + inclusionHeightOpt, + spendingTxIdOpt, + spendingHeightOpt, + box, + Set.empty[ScanId]), + confirmations + ) with BlockSection { + + override def parentId: ModifierId = null + override val modifierTypeId: ModifierTypeId = IndexedErgoBox.modifierTypeId + override val sizeOpt: Option[Int] = None + override def serializedId: Array[Byte] = box.id + override type M = IndexedErgoBox + override def serializer: ScorexSerializer[IndexedErgoBox] = IndexedErgoBoxSerializer + + def getAddress: ErgoAddress = box.ergoTree.root match { + case Right(_) => ExtraIndexerRef.getAddressEncoder.fromProposition(box.ergoTree).get // default most of the time + case Left(_) => new Pay2SAddress(box.ergoTree, box.ergoTree.bytes)(ExtraIndexerRef.getAddressEncoder) // needed for burn address 4MQyMKvMbnCJG3aJ + } + + def toBytes: Array[Byte] = serializer.toBytes(this) + + def asSpent(txId: ModifierId, txHeight: Int): IndexedErgoBox = + new IndexedErgoBox(inclusionHeightOpt, Some(txId), Some(txHeight), box, globalIndex, confirmations) +} +object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { + + override def serialize(iEb: IndexedErgoBox, w: Writer): Unit = { + w.putUByte(IndexedErgoBox.modifierTypeId) + w.putOption[Int](iEb.inclusionHeightOpt)(_.putInt(_)) + w.putOption[ModifierId](iEb.spendingTxIdOpt)((ww, id) => ww.putBytes(idToBytes(id))) + w.putOption[Int](iEb.spendingHeightOpt)(_.putInt(_)) + ErgoBoxSerializer.serialize(iEb.box, w) + w.putLong(iEb.globalIndex) + w.putOption[Int](iEb.confirmations)(_.putInt(_)) + } + + override def parse(r: Reader): IndexedErgoBox = { + val inclusionHeightOpt: Option[Int] = r.getOption[Int](r.getInt()) + val spendingTxIdOpt: Option[ModifierId] = r.getOption[ModifierId](bytesToId(r.getBytes(32))) + val spendingHeightOpt: Option[Int] = r.getOption[Int](r.getInt()) + val box: ErgoBox = ErgoBoxSerializer.parse(r) + val globalIndex: Long = r.getLong() + val confirmations: Option[Int] = r.getOption[Int](r.getInt()) + new IndexedErgoBox(inclusionHeightOpt, spendingTxIdOpt, spendingHeightOpt, box, globalIndex, confirmations) + } +} + +object IndexedErgoBox { + val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 5.toByte +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala new file mode 100644 index 0000000000..edf6645826 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -0,0 +1,87 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.modifiers.history.header.Header +import org.ergoplatform.modifiers.mempool.ErgoTransaction +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} +import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.{DataInput, JsonCodecs} +import scorex.core.serialization.ScorexSerializer +import scorex.core.{ModifierTypeId, idToBytes} +import scorex.util.serialization.{Reader, Writer} +import scorex.util.{ModifierId, bytesToId} + +case class IndexedErgoTransaction(txid: ModifierId, + height: Int, + globalIndex: Long) extends BlockSection with JsonCodecs { + + override val modifierTypeId: ModifierTypeId = IndexedErgoTransaction.modifierTypeId + override def serializedId: Array[Byte] = idToBytes(txid) + override val sizeOpt: Option[Int] = None + override def parentId: ModifierId = null + override type M = IndexedErgoTransaction + override def serializer: ScorexSerializer[IndexedErgoTransaction] = IndexedErgoTransactionSerializer + + def toBytes: Array[Byte] = serializer.toBytes(this) + + private var _blockId: ModifierId = _ + private var _inclusionHeight: Int = 0 + private var _timestamp: Header.Timestamp = 0L + private var _index: Int = 0 + private var _numConfirmations: Int = 0 + private var _inputs: IndexedSeq[IndexedErgoBox] = IndexedSeq.empty[IndexedErgoBox] + private var _dataInputs: IndexedSeq[DataInput] = IndexedSeq.empty[DataInput] + private var _outputs: IndexedSeq[IndexedErgoBox] = IndexedSeq.empty[IndexedErgoBox] + private var _txSize: Int = 0 + + def blockId: ModifierId = _blockId + def inclusionHeight: Int = _inclusionHeight + def timestamp: Header.Timestamp = _timestamp + def index: Int = _index + def numConfirmations: Int = _numConfirmations + def inputs: IndexedSeq[IndexedErgoBox] = _inputs + def dataInputs: IndexedSeq[DataInput] = _dataInputs + def outputs: IndexedSeq[IndexedErgoBox] = _outputs + def txSize: Int = _txSize + + def retrieveBody(history: ErgoHistoryReader): IndexedErgoTransaction = { + + val block: ErgoFullBlock = history.typedModifierById[Header](history.headerIdsAtHeight(height).head).flatMap(history.getFullBlock).get + val indexInBlock: Int = block.transactions.indices.find(block.transactions(_).id.equals(txid)).get + val tx: ErgoTransaction = block.transactions(indexInBlock) + + _blockId = block.id + _inclusionHeight = block.height + _timestamp = block.header.timestamp + _index = indexInBlock + _numConfirmations = history.bestFullBlockOpt.get.height - block.height + _inputs = tx.inputs.map(input => history.typedModifierById[IndexedErgoBox](bytesToId(input.boxId)).get) + _dataInputs = tx.dataInputs + _outputs = tx.outputs.map(output => history.typedModifierById[IndexedErgoBox](bytesToId(output.id)).get) + _txSize = tx.size + + this + } +} + +object IndexedErgoTransactionSerializer extends ScorexSerializer[IndexedErgoTransaction] { + + override def serialize(iTx: IndexedErgoTransaction, w: Writer): Unit = { + w.putUByte(IndexedErgoTransaction.modifierTypeId) + w.putUByte(iTx.serializedId.length) + w.putBytes(iTx.serializedId) + w.putInt(iTx.height) + w.putLong(iTx.globalIndex) + } + + override def parse(r: Reader): IndexedErgoTransaction = { + val idLen = r.getUByte() + val id = bytesToId(r.getBytes(idLen)) + val height = r.getInt() + val globalIndex = r.getLong() + IndexedErgoTransaction(id, height, globalIndex) + } +} + +object IndexedErgoTransaction { + val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 10.toByte +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala new file mode 100644 index 0000000000..58762927a1 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala @@ -0,0 +1,49 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.nodeView.history.ErgoHistoryReader +import scorex.core.{ModifierTypeId, idToBytes} +import scorex.core.serialization.ScorexSerializer +import scorex.util.{ModifierId, bytesToId} +import scorex.util.serialization.{Reader, Writer} + +case class IndexedErgoTree(treeHash: ModifierId, boxIds: Seq[ModifierId]) extends BlockSection { + override val sizeOpt: Option[Int] = None + override def serializedId: Array[Byte] = idToBytes(treeHash) + override def parentId: ModifierId = null + override val modifierTypeId: ModifierTypeId = IndexedErgoTree.modifierTypeId + override type M = IndexedErgoTree + override def serializer: ScorexSerializer[IndexedErgoTree] = IndexedErgoTreeSerializer + + def getBytes: Array[Byte] = serializer.toBytes(this) + + def retrieveBody(history: ErgoHistoryReader, lastN: Long): Seq[IndexedErgoBox] = + ( + if(lastN > 0) + boxIds.slice(math.max((boxIds.size - lastN).toInt, 0), boxIds.size) + else + boxIds + ).map(history.typedModifierById[IndexedErgoBox](_).get) +} + +object IndexedErgoTreeSerializer extends ScorexSerializer[IndexedErgoTree] { + + override def serialize(iEt: IndexedErgoTree, w: Writer): Unit = { + w.putUByte(IndexedErgoTree.modifierTypeId) + w.putBytes(iEt.serializedId) + w.putUInt(iEt.boxIds.length) + iEt.boxIds.foreach(id => w.putBytes(idToBytes(id))) + } + + override def parse(r: Reader): IndexedErgoTree = { + val treeHash: ModifierId = bytesToId(r.getBytes(32)) + val boxIdsLen: Long = r.getUInt() + var boxIds: Seq[ModifierId] = Seq.empty[ModifierId] + for(n <- 1L to boxIdsLen) boxIds = boxIds :+ bytesToId(r.getBytes(32)) + IndexedErgoTree(treeHash, boxIds) + } +} + +object IndexedErgoTree { + val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 20.toByte +} diff --git a/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala b/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala index 410f844601..d8eae05a50 100644 --- a/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala +++ b/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala @@ -45,6 +45,7 @@ case class NodeConfigurationSettings(stateType: StateType, minimalFeeAmount: Long, headerChainDiff: Int, adProofsSuffixLength: Int, + extraIndex: Boolean, blacklistedTransactions: Seq[String] = Seq.empty, checkpoint: Option[CheckpointSettings] = None) { /** @@ -81,6 +82,7 @@ trait NodeConfigurationReaders extends StateTypeReaders with CheckpointingSettin cfg.as[Long](s"$path.minimalFeeAmount"), cfg.as[Int](s"$path.headerChainDiff"), cfg.as[Int](s"$path.adProofsSuffixLength"), + cfg.as[Boolean](s"$path.extraIndex"), cfg.as[Seq[String]](s"$path.blacklistedTransactions"), cfg.as[Option[CheckpointSettings]](s"$path.checkpoint") ) From 79dc4bfe1eb29ab2dda62ea00f1ec2b93711ab11 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Tue, 16 Aug 2022 17:03:44 +0200 Subject: [PATCH 02/90] optimizations & bugfixes --- .../org/ergoplatform/http/api/ApiCodecs.scala | 10 +- .../http/api/ExtraIndexApiRoute.scala | 27 ++-- .../history/HistoryModifierSerializer.scala | 12 +- .../nodeView/history/extra/ExtraIndexer.scala | 136 ++++++++++-------- .../history/extra/IndexedErgoAddress.scala | 73 +++++----- .../history/extra/IndexedErgoBox.scala | 32 ++--- .../extra/IndexedErgoTransaction.scala | 7 +- .../history/extra/IndexedErgoTree.scala | 7 +- .../nodeView/history/extra/IndexedToken.scala | 60 ++++++++ .../nodeView/history/extra/NumericIndex.scala | 68 +++++++++ 10 files changed, 288 insertions(+), 144 deletions(-) create mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala create mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index 06419df406..981033f2f4 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -495,11 +495,11 @@ trait ApiCodecs extends JsonCodecs { implicit val indexedAddressEncoder: Encoder[IndexedErgoAddress] = { address => Json.obj( - "transactions" -> address.transactions(-1).map(_.asJson).asJson, - "txCount" -> address.transactions(-1).size.asJson, - "utxos" -> address.utxos(-1).asJson, - "balance" -> address.utxos(-1).map(_.box.value).sum.asJson, - "tokens" -> address.utxos(-1).map(_.box.additionalTokens.toArray.toSeq.asJson).asJson + "transactions" -> address.transactions().map(_.asJson).asJson, + "txCount" -> address.transactions().size.asJson, + "utxos" -> address.utxos().asJson, + "balance" -> address.utxos().map(_.box.value).sum.asJson, + "tokens" -> address.utxos().map(_.box.additionalTokens.toArray.toSeq.asJson).asJson ) } } diff --git a/src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala index 924356e5e8..0d167cf562 100644 --- a/src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala @@ -6,8 +6,7 @@ import akka.pattern.ask import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} import org.ergoplatform.nodeView.ErgoReadersHolder.GetDataFromHistory import org.ergoplatform.nodeView.history.ErgoHistoryReader -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{box_indexNumHash, ergoTreeHash, tx_indexNumHash} -import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoAddressSerializer, IndexedErgoBox, IndexedErgoTransaction, IndexedErgoTree} +import org.ergoplatform.nodeView.history.extra._ import org.ergoplatform.settings.ErgoSettings import scorex.core.api.http.ApiError.BadRequest import scorex.core.api.http.ApiResponse @@ -52,9 +51,7 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting (readersHolder ? GetDataFromHistory[ErgoHistoryReader](r => r)).mapTo[ErgoHistoryReader] private def getAddress(addr: ErgoAddress)(implicit history: ErgoHistoryReader): Option[IndexedErgoAddress] = { - val x: Option[IndexedErgoAddress] = history.typedModifierById[IndexedErgoAddress](IndexedErgoAddressSerializer.addressToModifierId(addr)) - if(x.isDefined) log.info(s"Address: ${addressEncoder.toString(x.get.address)} => ${x.get.txIds}, ${x.get.boxIds}") - x + history.typedModifierById[IndexedErgoAddress](IndexedErgoAddressSerializer.addressToModifierId(addr)) } private def getTxById(id: ModifierId)(implicit history: ErgoHistoryReader): Option[IndexedErgoTransaction] = @@ -75,7 +72,7 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getTxByIndex(index: Long): Future[Option[IndexedErgoTransaction]] = getHistory.map { history => - getTxById(bytesToId(history.modifierBytesById(bytesToId(tx_indexNumHash(index))).get))(history) // prepend 0 byte to circumvent "removing modifier type byte with .tail" + getTxById(history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(index))).get.m)(history) } private def getTxByIndexR: Route = (pathPrefix("transaction" / "byIndex" / LongNumber) & get) { index => @@ -85,7 +82,7 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getTxsByAddress(addr: ErgoAddress, lastN: Long): Future[Option[Seq[IndexedErgoTransaction]]] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => Some(addr.retrieveBody(history).transactions(lastN)) + case Some(addr) => Some(addr.retrieveTxs(history, lastN).transactions(-1)) case None => None } } @@ -100,7 +97,7 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getTxRange(fromHeight: Long, toHeight: Long): Future[Seq[ModifierId]] = getHistory.map { history => (for(n <- fromHeight to toHeight) - yield bytesToId(history.modifierBytesById(bytesToId(tx_indexNumHash(n))).get) + yield history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(n))).get.m ).toSeq } @@ -128,7 +125,7 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxByIndex(index: Long): Future[Option[IndexedErgoBox]] = getHistory.map { history => - getBoxById(bytesToId(history.modifierBytesById(bytesToId(box_indexNumHash(index))).get))(history) + getBoxById(history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(index))).get.m)(history) } private def getBoxByIndexR: Route = (pathPrefix("box" / "byIndex" / LongNumber) & get) { index => @@ -138,7 +135,7 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxesByAddress(addr: ErgoAddress, lastN: Long): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => addr.retrieveBody(history).boxes(lastN) + case Some(addr) => addr.retrieveBoxes(history, lastN).boxes() case None => Seq.empty[IndexedErgoBox] } } @@ -153,12 +150,12 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxesByAddressUnspent(addr: ErgoAddress, lastN: Long): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => addr.retrieveBody(history).utxos(lastN) + case Some(addr) => addr.retrieveBoxes(history, lastN).utxos() case None => Seq.empty[IndexedErgoBox] } } - private def getBoxesByAddressUnspentR: Route = (get & pathPrefix("transaction" / "unspent" / "byAddress") & path(Segment) & lastN) { (address, limit) => + private def getBoxesByAddressUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byAddress") & path(Segment) & lastN) { (address, limit) => addressEncoder.fromString(address) match { case Success(addr) => ApiResponse(getBoxesByAddressUnspent(addr, limit)) case Failure(_) => BadRequest("Incorrect address format") @@ -168,7 +165,7 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxRange(fromHeight: Long, toHeight: Long): Future[Seq[ModifierId]] = getHistory.map { history => (for(n <- fromHeight to toHeight) - yield bytesToId(history.modifierBytesById(bytesToId(box_indexNumHash(n))).get) + yield history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(n))).get.m ).toSeq } @@ -184,7 +181,7 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxesByErgoTree(tree: ErgoTree, lastN: Long): Future[Seq[IndexedErgoBox]] = getHistory.map { history => - history.typedModifierById[IndexedErgoTree](bytesToId(ergoTreeHash(tree))) match { + history.typedModifierById[IndexedErgoTree](bytesToId(IndexedErgoTreeSerializer.ergoTreeHash(tree))) match { case Some(iEt) => iEt.retrieveBody(history, lastN) case None => Seq.empty[IndexedErgoBox] } @@ -200,7 +197,7 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxesByErgoTreeUnspent(tree: ErgoTree, lastN: Long): Future[Seq[IndexedErgoBox]] = getHistory.map { history => - history.typedModifierById[IndexedErgoTree](bytesToId(ergoTreeHash(tree))) match { + history.typedModifierById[IndexedErgoTree](bytesToId(IndexedErgoTreeSerializer.ergoTreeHash(tree))) match { case Some(iEt) => iEt.retrieveBody(history, lastN).filter(!_.trackedBox.isSpent) case None => Seq.empty[IndexedErgoBox] } diff --git a/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala b/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala index ad29660c1b..1b54b8bbff 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala @@ -3,7 +3,7 @@ package org.ergoplatform.modifiers.history import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionSerializer} import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} -import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoAddressSerializer, IndexedErgoBox, IndexedErgoBoxSerializer, IndexedErgoTransaction, IndexedErgoTransactionSerializer, IndexedErgoTree, IndexedErgoTreeSerializer} +import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoAddressSerializer, IndexedErgoBox, IndexedErgoBoxSerializer, IndexedErgoTransaction, IndexedErgoTransactionSerializer, IndexedErgoTree, IndexedErgoTreeSerializer, NumericBoxIndex, NumericBoxIndexSerializer, NumericTxIndex, NumericTxIndexSerializer} import scorex.core.serialization.ScorexSerializer import scorex.util.serialization.{Reader, Writer} @@ -35,6 +35,12 @@ object HistoryModifierSerializer extends ScorexSerializer[BlockSection] { case m: IndexedErgoBox => w.put(IndexedErgoBox.modifierTypeId) IndexedErgoBoxSerializer.serialize(m, w) + case m: NumericTxIndex => + w.put(NumericTxIndex.modifierTypeId) + NumericTxIndexSerializer.serialize(m, w) + case m: NumericBoxIndex => + w.put(NumericBoxIndex.modifierTypeId) + NumericBoxIndexSerializer.serialize(m, w) case m => throw new Error(s"Serialization for unknown modifier: $m") } @@ -58,6 +64,10 @@ object HistoryModifierSerializer extends ScorexSerializer[BlockSection] { IndexedErgoTransactionSerializer.parse(r) case IndexedErgoBox.`modifierTypeId` => IndexedErgoBoxSerializer.parse(r) + case NumericTxIndex.`modifierTypeId` => + NumericTxIndexSerializer.parse(r) + case NumericBoxIndex.`modifierTypeId` => + NumericBoxIndexSerializer.parse(r) case m => throw new Error(s"Deserialization for unknown type byte: $m") } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 5871b4b73a..d65ab73bfd 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -3,32 +3,35 @@ package org.ergoplatform.nodeView.history.extra import akka.actor.{Actor, ActorRef, ActorSystem, Props} import org.ergoplatform.ErgoBox.BoxId import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} -import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} +import org.ergoplatform.ErgoAddressEncoder import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{box_indexNumHash, ergoTreeHash, tx_indexNumHash} import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.settings.{Algos, ChainSettings, ErgoAlgos} import scorex.db.ByteArrayWrapper -import scorex.util.{ScorexLogging, bytesToId} -import sigmastate.Values.ErgoTree +import scorex.util.{ModifierId, ScorexLogging, bytesToId} import java.nio.ByteBuffer +import scala.collection.mutable import scala.collection.mutable.ArrayBuffer +import scala.reflect.ClassTag class ExtraIndex(chainSettings: ChainSettings) extends Actor with ScorexLogging { private val IndexedHeightKey: ByteArrayWrapper = ByteArrayWrapper.apply(Algos.hash("indexed height")) private var indexedHeight: Int = 0 + private val indexedHeightBuffer : ByteBuffer = ByteBuffer.allocate(4) private val GlobalTxIndexKey: ByteArrayWrapper = ByteArrayWrapper.apply(Algos.hash("txns height")) private var globalTxIndex: Long = 0L + private val globalTxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) private val GlobalBoxIndexKey: ByteArrayWrapper = ByteArrayWrapper.apply(Algos.hash("boxes height")) private var globalBoxIndex: Long = 0L + private val globalBoxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) private var done: Boolean = false @@ -38,76 +41,95 @@ class ExtraIndex(chainSettings: ChainSettings) private def history: ErgoHistoryReader = _history.asInstanceOf[ErgoHistoryReader] private def historyStorage: HistoryStorage = _history.historyStorage + // fast access + private val modifiers: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] + private val boxesGroupedByAddress: mutable.HashMap[ModifierId, Seq[BoxId]] = mutable.HashMap.empty[ModifierId, Seq[BoxId]] + private val emptyBoxSeq: Seq[BoxId] = Seq.empty[BoxId] + + // if any element "e" of type "T" satisfies "f(e) == true", the element and its index in the buffer are returned + private def updateMatch[T <: BlockSection : ClassTag](f: T => Boolean): Option[(T, Int)] = { + modifiers.indices.foreach({ n => + modifiers(n) match { + case e: T if f(e) => return Some((e, n)) + case _ => + } + }) + None + } + private def index(bt: BlockTransactions, height: Int) = { - var txIndexTip: IndexedErgoTransaction = null - var boxIndexTip: IndexedErgoBox = null + modifiers.clear() + boxesGroupedByAddress.clear() //process transactions bt.txIds.indices.foreach(n => { - val tx = IndexedErgoTransaction(bytesToId(bt.txIds(n)), height, globalTxIndex) - historyStorage.insert(bt.txIds(n), tx.toBytes) // tx by id - historyStorage.insert(tx_indexNumHash(globalTxIndex), 0.toByte +: bt.txIds(n)) // tx id by global tx number (repend 0 byte to circumvent "removing modifier type byte with .tail") + modifiers += IndexedErgoTransaction(bytesToId(bt.txIds(n)), indexedHeight, globalTxIndex) + modifiers += new NumericTxIndex(globalTxIndex, bytesToId(bt.txIds(n))) globalTxIndex += 1 - txIndexTip = tx }) bt.txs.foreach(tx => { //process tx inputs - if(height != 1) { //only after 1st block (skip genesis box) - tx.inputs.indices.foreach(n => { - val id: BoxId = tx.inputs(n).boxId - history.typedModifierById[IndexedErgoBox](bytesToId(id)) match { - case Some(iEb) => - boxIndexTip = iEb.asSpent(tx.id, height) - historyStorage.insert(id, boxIndexTip.toBytes) // box by id - case None => log.warn(s"Input for box ${ErgoAlgos.encode(id)} not found in database (this shouldn't happen)") + if(indexedHeight != 1) { //only after 1st block (skip genesis box) + tx.inputs.foreach(in => + updateMatch[IndexedErgoBox](x => java.util.Arrays.equals(x.box.id, in.boxId)) match { + case Some((iEb, n)) => modifiers.update(n, iEb.asSpent(tx.id, indexedHeight)) // box found in this block, update + case None => // box not found in this block + history.typedModifierById[IndexedErgoBox](bytesToId(in.boxId)) match { + case Some(x) => modifiers += x.asSpent(tx.id, indexedHeight) // box found in DB, update + case None => log.warn(s"Input for box ${ErgoAlgos.encode(in.boxId)} not found in database") // box not found at all (this shouldn't happen) + } } - }) + ) } //process tx outputs - var outputs: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] - tx.outputs.indices.foreach(n => { - boxIndexTip = new IndexedErgoBox(Some(height), None, None, tx.outputs(n), globalBoxIndex, Some(chainHeight - height)) - historyStorage.insert(boxIndexTip.serializedId, boxIndexTip.toBytes) // box by id - historyStorage.insert(box_indexNumHash(globalBoxIndex), 0.toByte +: boxIndexTip.serializedId) // box id by global box number (repend 0 byte to circumvent "removing modifier type byte with .tail") - val iEt: IndexedErgoTree = history.typedModifierById[IndexedErgoTree](bytesToId(ergoTreeHash(boxIndexTip.box.ergoTree))) match { - case Some(x) => IndexedErgoTree(x.treeHash, x.boxIds :+ bytesToId(boxIndexTip.box.id)) // ergotree found, update - case None => IndexedErgoTree(bytesToId(ergoTreeHash(boxIndexTip.box.ergoTree)), Seq(bytesToId(boxIndexTip.box.id))) // ergotree not found, record + tx.outputs.foreach(box => { + modifiers += new IndexedErgoBox(Some(indexedHeight), None, None, box, globalBoxIndex, Some(chainHeight - indexedHeight)) // box by id + modifiers += new NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number + val maybeNewTree: ModifierId = bytesToId(IndexedErgoTreeSerializer.ergoTreeHash(box.ergoTree)) + updateMatch[IndexedErgoTree](_.treeHash.eq(maybeNewTree)) match { + case Some((iEt, n)) => modifiers.update(n, IndexedErgoTree(iEt.treeHash, iEt.boxIds :+ bytesToId(box.id))) // ergotree found in this block, update + case None => // ergotree not found in this TX + history.typedModifierById[IndexedErgoTree](maybeNewTree) match { + case Some(x) => modifiers += IndexedErgoTree(x.treeHash, x.boxIds :+ bytesToId(box.id)) // ergotree found in DB, update + case None => modifiers += IndexedErgoTree(maybeNewTree, Seq(bytesToId(box.id))) // ergotree not found at all, record + } } - historyStorage.insert(iEt.serializedId, iEt.getBytes) // box id by ergotree - outputs :+= boxIndexTip globalBoxIndex += 1 + val addrHash: ModifierId = IndexedErgoAddressSerializer.addressToModifierId(IndexedErgoBoxSerializer.getAddress(box.ergoTree)) + boxesGroupedByAddress.put(addrHash, boxesGroupedByAddress.getOrElse[Seq[BoxId]](addrHash, emptyBoxSeq) :+ box.id) }) //process boxes by address - val boxesGroupedByAddress : Array[(ErgoAddress, Seq[BoxId])] = outputs.map(x => (x.getAddress, x.box.id)).groupBy(_._1).mapValues(_.map(_._2).toSeq).toArray - for(addressWithBoxes <- boxesGroupedByAddress) { - val addr: IndexedErgoAddress = - history.typedModifierById[IndexedErgoAddress](IndexedErgoAddressSerializer.addressToModifierId(addressWithBoxes._1)) match { - case Some(iEa) => new IndexedErgoAddress(addressWithBoxes._1, iEa.txIds :+ tx.id, iEa.boxIds ++ addressWithBoxes._2) //address found, update - case None => new IndexedErgoAddress(addressWithBoxes._1, Seq(tx.id), addressWithBoxes._2) //address not found, record - } - historyStorage.insert(addr.serializedId, addr.getBytes) - } - + for(addressWithBoxes <- boxesGroupedByAddress) + updateMatch[IndexedErgoAddress](_.addressHash.eq(addressWithBoxes._1)) match { + case Some((iEa, n)) => modifiers.update(n, IndexedErgoAddress(addressWithBoxes._1, iEa.txIds :+ tx.id, iEa.boxIds ++ addressWithBoxes._2)) // address found in this block, update + case None => // address not found in this block + history.typedModifierById[IndexedErgoAddress](addressWithBoxes._1) match { + case Some(x) => modifiers += IndexedErgoAddress(addressWithBoxes._1, x.txIds :+ tx.id, x.boxIds ++ addressWithBoxes._2) //address found in DB, update + case None => modifiers += IndexedErgoAddress(addressWithBoxes._1, Seq(tx.id), addressWithBoxes._2) //address not found at all, record + } + } }) - log.info(s"Indexed block #$height [transactions: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] - progress: $height / $chainHeight") - if(done) indexedHeight = height // after the indexer caught up with the chain height gets updated here - writeProgress() - } + log.info(s"Indexed block #$indexedHeight [transactions: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] - progress: $indexedHeight / $chainHeight") + + if(done) indexedHeight = height // update after caught up with chain + + indexedHeightBuffer.clear() + globalTxIndexBuffer.clear() + globalBoxIndexBuffer.clear() + + historyStorage.insert(Seq((IndexedHeightKey , indexedHeightBuffer .putInt (indexedHeight ).array), + (GlobalTxIndexKey , globalTxIndexBuffer .putLong(globalTxIndex ).array), + (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), modifiers) - private val empty: Seq[BlockSection] = Seq.empty[BlockSection] - private def writeProgress() = { - historyStorage.insert(Seq((IndexedHeightKey , ByteBuffer.allocate(4).putInt (indexedHeight ).array), - (GlobalTxIndexKey , ByteBuffer.allocate(8).putLong(globalTxIndex ).array), - (GlobalBoxIndexKey, ByteBuffer.allocate(8).putLong(globalBoxIndex).array)), empty) } - private def run() = { + private def run(): Unit = { indexedHeight = ByteBuffer.wrap(historyStorage.getIndex(IndexedHeightKey ).getOrElse(Array.fill[Byte](4){0})).getInt globalTxIndex = ByteBuffer.wrap(historyStorage.getIndex(GlobalTxIndexKey ).getOrElse(Array.fill[Byte](8){0})).getLong @@ -130,12 +152,13 @@ class ExtraIndex(chainSettings: ChainSettings) } override def postStop(): Unit = { - if(indexedHeight != 0) // only after done or stopped, not at shutdown - log.info(s"Stopped extra indexer at height ${indexedHeight - 1}") + log.info(s"Stopped extra indexer at height ${indexedHeight - 1}") } override def receive: Receive = { - case SemanticallySuccessfulModifier(fb: ErgoFullBlock) => if(done) index(fb.blockTransactions, fb.height) // after the indexer caught up with the chain, stay up to date + case SemanticallySuccessfulModifier(fb: ErgoFullBlock) => + if(done) // after the indexer caught up with the chain, stay up to date + index(fb.blockTransactions, fb.height) case Start(history: ErgoHistory) => _history = history run() @@ -160,11 +183,6 @@ object ExtraIndexerRef { ExtraIndexerRefHolder.init(actor) actor } - - def tx_indexNumHash(index: Long): Array[Byte] = Algos.hash("txns height " + index) - def box_indexNumHash(index: Long): Array[Byte] = Algos.hash("boxes height " + index) - - def ergoTreeHash(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) } object ExtraIndexerRefHolder { @@ -172,10 +190,10 @@ object ExtraIndexerRefHolder { private var _actor: Option[ActorRef] = None private var running: Boolean = false - def start(history: ErgoHistory) = if(_actor.isDefined && !running) { + def start(history: ErgoHistory): Unit = if(_actor.isDefined && !running) { _actor.get ! Start(history) running = true } - protected[extra] def init(actor: ActorRef) = _actor = Some(actor) + protected[extra] def init(actor: ActorRef): Unit = _actor = Some(actor) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 8ba32f827b..8aa6f51711 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -1,6 +1,6 @@ package org.ergoplatform.nodeView.history.extra -import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, P2PKAddress, Pay2SAddress, Pay2SHAddress} +import org.ergoplatform.ErgoAddress import org.ergoplatform.ErgoAddressEncoder.{ChecksumLength, hash256} import org.ergoplatform.ErgoBox.BoxId import org.ergoplatform.modifiers.BlockSection @@ -9,24 +9,20 @@ import org.ergoplatform.settings.Algos import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.crypto.authds.ADKey -import scorex.util.{ByteArrayOps, ModifierId, bytesToId} +import scorex.util.{ByteArrayOps, ModifierId, bytesToId, idToBytes} import scorex.util.serialization.{Reader, Writer} -import sigmastate.basics.DLogProtocol.ProveDlog -import sigmastate.serialization.{ErgoTreeSerializer, GroupElementSerializer, SigmaSerializer} -class IndexedErgoAddress(val address: ErgoAddress, - val txIds: Seq[ModifierId], - val boxIds: Seq[BoxId]) extends BlockSection { +case class IndexedErgoAddress(addressHash: ModifierId, + txIds: Seq[ModifierId], + boxIds: Seq[BoxId]) extends BlockSection { override val sizeOpt: Option[Int] = None - override def serializedId: Array[Byte] = Algos.hash(IndexedErgoAddressSerializer.addressToBytes(address)) + override def serializedId: Array[Byte] = idToBytes(addressHash) override def parentId: ModifierId = null override val modifierTypeId: ModifierTypeId = IndexedErgoAddress.modifierTypeId override type M = IndexedErgoAddress override def serializer: ScorexSerializer[IndexedErgoAddress] = IndexedErgoAddressSerializer - def getBytes: Array[Byte] = serializer.toBytes(this) - private var _transactions: Seq[IndexedErgoTransaction] = Seq.empty[IndexedErgoTransaction] private var _boxes: Seq[IndexedErgoBox] = Seq.empty[IndexedErgoBox] private var _utxos: Seq[IndexedErgoBox] = _ @@ -37,22 +33,33 @@ class IndexedErgoAddress(val address: ErgoAddress, else _transactions - def boxes(lastN: Long = 20L): Seq[IndexedErgoBox] = - if(lastN > 0) - _boxes.slice(math.max((_boxes.size - lastN).toInt, 0), _boxes.size) - else - _boxes + def boxes(): Seq[IndexedErgoBox] = _boxes - def utxos(lastN: Long = 20L): Seq[IndexedErgoBox] = { - if(lastN > 0) - _utxos.slice(math.max((_utxos.size - lastN).toInt, 0), _utxos.size) - else - _utxos - } + def utxos(): Seq[IndexedErgoBox] = _utxos def retrieveBody(history: ErgoHistoryReader): IndexedErgoAddress = { - _transactions = txIds.map(history.typedModifierById[IndexedErgoTransaction](_).get.retrieveBody(history)) - _boxes = boxIds.map(x => history.typedModifierById[IndexedErgoBox](bytesToId(x)).get) + retrieveTxs(history, -1) + retrieveBoxes(history, -1) + this + } + + def retrieveTxs(history: ErgoHistoryReader, lastN: Long): IndexedErgoAddress = { + _transactions = ( + if(lastN > 0) + txIds.slice(math.max((txIds.size - lastN).toInt, 0), txIds.size) + else + txIds + ).map(history.typedModifierById[IndexedErgoTransaction](_).get.retrieveBody(history)) + this + } + + def retrieveBoxes(history: ErgoHistoryReader, lastN: Long): IndexedErgoAddress = { + _boxes = ( + if(lastN > 0) + boxIds.slice(math.max((boxIds.size - lastN).toInt, 0), boxIds.size) + else + boxIds + ).map(x => history.typedModifierById[IndexedErgoBox](bytesToId(x)).get) _utxos = _boxes.filter(!_.trackedBox.isSpent) this } @@ -67,21 +74,8 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] def addressToModifierId(address: ErgoAddress): ModifierId = bytesToId(Algos.hash(addressToBytes(address))) - def bytesToAddress(bytes: Array[Byte]): ErgoAddress = { - val contentBytes: Array[Byte] = bytes.slice(1, bytes.length - ChecksumLength) - implicit val encoder: ErgoAddressEncoder = ExtraIndexerRef.getAddressEncoder - (bytes(0) & 0x0F).toByte match { - case P2PKAddress. addressTypePrefix => new P2PKAddress(ProveDlog(GroupElementSerializer.parseTry(SigmaSerializer.startReader(contentBytes)).get), contentBytes) - case Pay2SHAddress.addressTypePrefix => new Pay2SHAddress(contentBytes) - case Pay2SAddress. addressTypePrefix => new Pay2SAddress(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(contentBytes), contentBytes) - } - } - override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = { - w.putUByte(IndexedErgoAddress.modifierTypeId) - val addressBytes: Array[Byte] = addressToBytes(iEa.address) - w.putUShort(addressBytes.length) - w.putBytes(addressBytes) + w.putBytes(idToBytes(iEa.addressHash)) w.putUInt(iEa.txIds.length) iEa.txIds.foreach(m => w.putBytes(m.toBytes)) w.putUInt(iEa.boxIds.length) @@ -89,15 +83,14 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] } override def parse(r: Reader): IndexedErgoAddress = { - val addressLen: Int = r.getUShort() - val address: ErgoAddress = bytesToAddress(r.getBytes(addressLen)) + val addressHash: ModifierId = bytesToId(r.getBytes(32)) val txnsLen: Long = r.getUInt() var txns: Seq[ModifierId] = Seq.empty[ModifierId] for(n <- 1L to txnsLen) txns = txns :+ r.getBytes(32).toModifierId val boxesLen: Long = r.getUInt() var boxes: Seq[BoxId] = Seq.empty[BoxId] for(n <- 1L to boxesLen) boxes = boxes :+ ADKey @@ r.getBytes(32) - new IndexedErgoAddress(address, txns, boxes) + new IndexedErgoAddress(addressHash, txns, boxes) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala index 70af3f839a..c1a6b1deac 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -9,6 +9,7 @@ import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId, idToBytes} import scorex.util.serialization.{Reader, Writer} +import sigmastate.Values.ErgoTree class IndexedErgoBox(val inclusionHeightOpt: Option[Int], val spendingTxIdOpt: Option[ModifierId], @@ -16,15 +17,14 @@ class IndexedErgoBox(val inclusionHeightOpt: Option[Int], val box: ErgoBox, val globalIndex: Long, val confirmations: Option[Int]) - extends WalletBox(new TrackedBox(box.transactionId, - box.index, - inclusionHeightOpt, - spendingTxIdOpt, - spendingHeightOpt, - box, - Set.empty[ScanId]), - confirmations - ) with BlockSection { + extends WalletBox(TrackedBox(box.transactionId, + box.index, + inclusionHeightOpt, + spendingTxIdOpt, + spendingHeightOpt, + box, + Set.empty[ScanId]), + confirmations) with BlockSection { override def parentId: ModifierId = null override val modifierTypeId: ModifierTypeId = IndexedErgoBox.modifierTypeId @@ -33,20 +33,20 @@ class IndexedErgoBox(val inclusionHeightOpt: Option[Int], override type M = IndexedErgoBox override def serializer: ScorexSerializer[IndexedErgoBox] = IndexedErgoBoxSerializer - def getAddress: ErgoAddress = box.ergoTree.root match { - case Right(_) => ExtraIndexerRef.getAddressEncoder.fromProposition(box.ergoTree).get // default most of the time - case Left(_) => new Pay2SAddress(box.ergoTree, box.ergoTree.bytes)(ExtraIndexerRef.getAddressEncoder) // needed for burn address 4MQyMKvMbnCJG3aJ - } - - def toBytes: Array[Byte] = serializer.toBytes(this) + def getAddress: ErgoAddress = IndexedErgoBoxSerializer.getAddress(box.ergoTree) def asSpent(txId: ModifierId, txHeight: Int): IndexedErgoBox = new IndexedErgoBox(inclusionHeightOpt, Some(txId), Some(txHeight), box, globalIndex, confirmations) } object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { + def getAddress(tree: ErgoTree): ErgoAddress = + tree.root match { + case Right(_) => ExtraIndexerRef.getAddressEncoder.fromProposition(tree).get // default most of the time + case Left(_) => new Pay2SAddress(tree, tree.bytes)(ExtraIndexerRef.getAddressEncoder) // needed for burn address 4MQyMKvMbnCJG3aJ + } + override def serialize(iEb: IndexedErgoBox, w: Writer): Unit = { - w.putUByte(IndexedErgoBox.modifierTypeId) w.putOption[Int](iEb.inclusionHeightOpt)(_.putInt(_)) w.putOption[ModifierId](iEb.spendingTxIdOpt)((ww, id) => ww.putBytes(idToBytes(id))) w.putOption[Int](iEb.spendingHeightOpt)(_.putInt(_)) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index edf6645826..2ea2e003e9 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -4,7 +4,7 @@ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} import org.ergoplatform.nodeView.history.ErgoHistoryReader -import org.ergoplatform.{DataInput, JsonCodecs} +import org.ergoplatform.DataInput import scorex.core.serialization.ScorexSerializer import scorex.core.{ModifierTypeId, idToBytes} import scorex.util.serialization.{Reader, Writer} @@ -12,7 +12,7 @@ import scorex.util.{ModifierId, bytesToId} case class IndexedErgoTransaction(txid: ModifierId, height: Int, - globalIndex: Long) extends BlockSection with JsonCodecs { + globalIndex: Long) extends BlockSection { override val modifierTypeId: ModifierTypeId = IndexedErgoTransaction.modifierTypeId override def serializedId: Array[Byte] = idToBytes(txid) @@ -21,8 +21,6 @@ case class IndexedErgoTransaction(txid: ModifierId, override type M = IndexedErgoTransaction override def serializer: ScorexSerializer[IndexedErgoTransaction] = IndexedErgoTransactionSerializer - def toBytes: Array[Byte] = serializer.toBytes(this) - private var _blockId: ModifierId = _ private var _inclusionHeight: Int = 0 private var _timestamp: Header.Timestamp = 0L @@ -66,7 +64,6 @@ case class IndexedErgoTransaction(txid: ModifierId, object IndexedErgoTransactionSerializer extends ScorexSerializer[IndexedErgoTransaction] { override def serialize(iTx: IndexedErgoTransaction, w: Writer): Unit = { - w.putUByte(IndexedErgoTransaction.modifierTypeId) w.putUByte(iTx.serializedId.length) w.putBytes(iTx.serializedId) w.putInt(iTx.height) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala index 58762927a1..1c0de445bf 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala @@ -2,10 +2,12 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.settings.Algos import scorex.core.{ModifierTypeId, idToBytes} import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} +import sigmastate.Values.ErgoTree case class IndexedErgoTree(treeHash: ModifierId, boxIds: Seq[ModifierId]) extends BlockSection { override val sizeOpt: Option[Int] = None @@ -15,8 +17,6 @@ case class IndexedErgoTree(treeHash: ModifierId, boxIds: Seq[ModifierId]) extend override type M = IndexedErgoTree override def serializer: ScorexSerializer[IndexedErgoTree] = IndexedErgoTreeSerializer - def getBytes: Array[Byte] = serializer.toBytes(this) - def retrieveBody(history: ErgoHistoryReader, lastN: Long): Seq[IndexedErgoBox] = ( if(lastN > 0) @@ -28,8 +28,9 @@ case class IndexedErgoTree(treeHash: ModifierId, boxIds: Seq[ModifierId]) extend object IndexedErgoTreeSerializer extends ScorexSerializer[IndexedErgoTree] { + def ergoTreeHash(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) + override def serialize(iEt: IndexedErgoTree, w: Writer): Unit = { - w.putUByte(IndexedErgoTree.modifierTypeId) w.putBytes(iEt.serializedId) w.putUInt(iEt.boxIds.length) iEt.boxIds.foreach(id => w.putBytes(idToBytes(id))) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala new file mode 100644 index 0000000000..e3db27a962 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -0,0 +1,60 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.ErgoBox +import org.ergoplatform.ErgoBox.{R4, R5, R6} +import org.ergoplatform.modifiers.BlockSection +import scorex.core.{ModifierTypeId, idToBytes} +import scorex.core.serialization.ScorexSerializer +import scorex.util.{ModifierId, bytesToId} +import scorex.util.serialization.{Reader, Writer} +import sigmastate.SByte +import sigmastate.Values.CollectionConstant + +case class IndexedToken(tokenId: ModifierId, + amount: Long, + name: String, + description: String, + decimals: Byte) extends BlockSection { + override def parentId: ModifierId = null + override val modifierTypeId: ModifierTypeId = IndexedToken.modifierTypeId + override type M = IndexedToken + override def serializer: ScorexSerializer[IndexedToken] = IndexedTokenSerializer + override val sizeOpt: Option[Int] = None + override def serializedId: Array[Byte] = idToBytes(tokenId) +} + +object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { + + def hasTokenEmission(box: ErgoBox): Boolean = + box.additionalTokens.size >= 1 && box.additionalRegisters.contains(R4) && box.additionalRegisters.contains(R5) && box.additionalRegisters.contains(R6) + + def fromBox(box: ErgoBox): IndexedToken = + IndexedToken(bytesToId(box.additionalTokens(0)._1), + box.additionalTokens(0)._2, + new String(box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]].items.asInstanceOf[Seq[Byte]].toArray, "UTF-8"), + new String(box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]].items.asInstanceOf[Seq[Byte]].toArray, "UTF-8"), + box.additionalRegisters(R6).asInstanceOf[CollectionConstant[SByte.type]].items.asInstanceOf[Seq[Byte]].head) + + override def serialize(iT: IndexedToken, w: Writer): Unit = { + w.putBytes(iT.serializedId) + w.putULong(iT.amount) + w.putUShort(iT.name.length) + w.putBytes(iT.name.getBytes("UTF-8")) + w.putUShort(iT.description.length) + w.putBytes(iT.description.getBytes("UTF-8")) + w.putUByte(iT.decimals) + } + + override def parse(r: Reader): IndexedToken = { + val tokenId: ModifierId = bytesToId(r.getBytes(32)) + val amount: Long = r.getULong() + val name: String = new String(r.getBytes(r.getUShort), "UTF-8") + val description: String = new String(r.getBytes(r.getUShort), "UTF-8") + val decimals: Byte = r.getUByte().toByte + IndexedToken(tokenId, amount, name, description, decimals) + } +} + +object IndexedToken { + val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 35.toByte +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala new file mode 100644 index 0000000000..4dda4d988a --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala @@ -0,0 +1,68 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.settings.Algos +import scorex.core.{ModifierTypeId, idToBytes} +import scorex.core.serialization.ScorexSerializer +import scorex.util.{ModifierId, bytesToId} +import scorex.util.serialization.{Reader, Writer} + +case class NumericTxIndex(n: Long, m: ModifierId) extends BlockSection { + override val sizeOpt: Option[Int] = None + override def serializedId: Array[Byte] = NumericTxIndex.indexToBytes(n) + override def parentId: ModifierId = null + override val modifierTypeId: ModifierTypeId = NumericTxIndex.modifierTypeId + override type M = NumericTxIndex + override def serializer: ScorexSerializer[NumericTxIndex] = NumericTxIndexSerializer + + def hashValue: Array[Byte] = Array.empty[Byte] +} +object NumericTxIndexSerializer extends ScorexSerializer[NumericTxIndex] { + + override def serialize(ni: NumericTxIndex, w: Writer): Unit = { + w.putUInt(ni.n) + w.putBytes(idToBytes(ni.m)) + } + + override def parse(r: Reader): NumericTxIndex = { + val n: Long = r.getUInt() + val m: ModifierId = bytesToId(r.getBytes(32)) + NumericTxIndex(n, m) + } +} + +object NumericTxIndex { + val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 25.toByte + + def indexToBytes(n: Long): Array[Byte] = Algos.hash("txns height " + n) +} + +case class NumericBoxIndex(n: Long, m: ModifierId) extends BlockSection { + override val sizeOpt: Option[Int] = None + override def serializedId: Array[Byte] = NumericBoxIndex.indexToBytes(n) + override def parentId: ModifierId = null + override val modifierTypeId: ModifierTypeId = NumericBoxIndex.modifierTypeId + override type M = NumericBoxIndex + override def serializer: ScorexSerializer[NumericBoxIndex] = NumericBoxIndexSerializer + + def hashValue: Array[Byte] = Array.empty[Byte] +} +object NumericBoxIndexSerializer extends ScorexSerializer[NumericBoxIndex] { + + override def serialize(ni: NumericBoxIndex, w: Writer): Unit = { + w.putUInt(ni.n) + w.putBytes(idToBytes(ni.m)) + } + + override def parse(r: Reader): NumericBoxIndex = { + val n: Long = r.getUInt() + val m: ModifierId = bytesToId(r.getBytes(32)) + NumericBoxIndex(n, m) + } +} + +object NumericBoxIndex { + val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 30.toByte + + def indexToBytes(n: Long): Array[Byte] = Algos.hash("boxes height " + n) +} From 8531a56f29e0f1cb9d2f5c88797acf0fbbcf5206 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Fri, 19 Aug 2022 00:12:34 +0200 Subject: [PATCH 03/90] more optimizations --- src/main/resources/application.conf | 3 + .../nodeView/history/extra/ExtraIndexer.scala | 92 +++++++++---------- .../history/extra/IndexedErgoAddress.scala | 28 ++++-- .../history/extra/IndexedErgoBox.scala | 11 ++- .../history/extra/IndexedErgoTree.scala | 13 ++- .../history/storage/HistoryStorage.scala | 9 +- .../ergoplatform/settings/CacheSettings.scala | 2 +- .../settings/ErgoSettingsSpecification.scala | 15 +-- 8 files changed, 105 insertions(+), 68 deletions(-) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index e3f5f7e24f..57624577b4 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -110,6 +110,9 @@ ergo { # Number of recently used block sections that will be kept in memory blockSectionsCacheSize = 12 + # Number of recently used extra indexes that will be kept in memory (has no effect if extraIndex is disabled) + extraCacheSize = 250 + # Number of recently used headers that will be kept in memory headersCacheSize = 1000 diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index d65ab73bfd..d2ad9dc89c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -11,12 +11,11 @@ import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessage import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.settings.{Algos, ChainSettings, ErgoAlgos} import scorex.db.ByteArrayWrapper -import scorex.util.{ModifierId, ScorexLogging, bytesToId} +import scorex.util.{ScorexLogging, bytesToId} import java.nio.ByteBuffer import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer -import scala.reflect.ClassTag +import scala.collection.mutable.{ArrayBuffer, ListBuffer} class ExtraIndex(chainSettings: ChainSettings) extends Actor with ScorexLogging { @@ -33,6 +32,8 @@ class ExtraIndex(chainSettings: ChainSettings) private var globalBoxIndex: Long = 0L private val globalBoxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) + private val saveLimit: Int = 10 + private var done: Boolean = false private def chainHeight: Int = _history.fullBlockHeight @@ -43,29 +44,34 @@ class ExtraIndex(chainSettings: ChainSettings) // fast access private val modifiers: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] - private val boxesGroupedByAddress: mutable.HashMap[ModifierId, Seq[BoxId]] = mutable.HashMap.empty[ModifierId, Seq[BoxId]] - private val emptyBoxSeq: Seq[BoxId] = Seq.empty[BoxId] - - // if any element "e" of type "T" satisfies "f(e) == true", the element and its index in the buffer are returned - private def updateMatch[T <: BlockSection : ClassTag](f: T => Boolean): Option[(T, Int)] = { - modifiers.indices.foreach({ n => - modifiers(n) match { - case e: T if f(e) => return Some((e, n)) - case _ => - } - }) - None - } + private val boxesGroupedByAddress: mutable.HashMap[Array[Byte], ListBuffer[BoxId]] = mutable.HashMap.empty[Array[Byte], ListBuffer[BoxId]] + private val emptyBoxBuffer: ListBuffer[BoxId] = ListBuffer.empty[BoxId] + + private def findModifierOpt(id: Array[Byte]): Option[Int] = + modifiers.indexWhere(x => java.util.Arrays.equals(x.serializedId, id)) match { + case x: Int if x >= 0 => Some(x) + case _ => None + } - private def index(bt: BlockTransactions, height: Int) = { + private def saveProgress(): Unit = { + indexedHeightBuffer.clear() + globalTxIndexBuffer.clear() + globalBoxIndexBuffer.clear() + + historyStorage.insert(Seq((IndexedHeightKey , indexedHeightBuffer .putInt (indexedHeight ).array), + (GlobalTxIndexKey , globalTxIndexBuffer .putLong(globalTxIndex ).array), + (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), modifiers) modifiers.clear() boxesGroupedByAddress.clear() + } + + private def index(bt: BlockTransactions, height: Int): Unit = { //process transactions - bt.txIds.indices.foreach(n => { - modifiers += IndexedErgoTransaction(bytesToId(bt.txIds(n)), indexedHeight, globalTxIndex) - modifiers += new NumericTxIndex(globalTxIndex, bytesToId(bt.txIds(n))) + bt.txIds.foreach(id => { + modifiers += IndexedErgoTransaction(bytesToId(id), indexedHeight, globalTxIndex) + modifiers += NumericTxIndex(globalTxIndex, bytesToId(id)) globalTxIndex += 1 }) @@ -74,9 +80,9 @@ class ExtraIndex(chainSettings: ChainSettings) //process tx inputs if(indexedHeight != 1) { //only after 1st block (skip genesis box) tx.inputs.foreach(in => - updateMatch[IndexedErgoBox](x => java.util.Arrays.equals(x.box.id, in.boxId)) match { - case Some((iEb, n)) => modifiers.update(n, iEb.asSpent(tx.id, indexedHeight)) // box found in this block, update - case None => // box not found in this block + findModifierOpt(in.boxId) match { + case Some(n) => modifiers(n).asInstanceOf[IndexedErgoBox].asSpent(tx.id, indexedHeight) // box found in last saveLimit blocks, update + case None => // box not found in this block history.typedModifierById[IndexedErgoBox](bytesToId(in.boxId)) match { case Some(x) => modifiers += x.asSpent(tx.id, indexedHeight) // box found in DB, update case None => log.warn(s"Input for box ${ErgoAlgos.encode(in.boxId)} not found in database") // box not found at all (this shouldn't happen) @@ -88,29 +94,29 @@ class ExtraIndex(chainSettings: ChainSettings) //process tx outputs tx.outputs.foreach(box => { modifiers += new IndexedErgoBox(Some(indexedHeight), None, None, box, globalBoxIndex, Some(chainHeight - indexedHeight)) // box by id - modifiers += new NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number - val maybeNewTree: ModifierId = bytesToId(IndexedErgoTreeSerializer.ergoTreeHash(box.ergoTree)) - updateMatch[IndexedErgoTree](_.treeHash.eq(maybeNewTree)) match { - case Some((iEt, n)) => modifiers.update(n, IndexedErgoTree(iEt.treeHash, iEt.boxIds :+ bytesToId(box.id))) // ergotree found in this block, update - case None => // ergotree not found in this TX - history.typedModifierById[IndexedErgoTree](maybeNewTree) match { - case Some(x) => modifiers += IndexedErgoTree(x.treeHash, x.boxIds :+ bytesToId(box.id)) // ergotree found in DB, update - case None => modifiers += IndexedErgoTree(maybeNewTree, Seq(bytesToId(box.id))) // ergotree not found at all, record + modifiers += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number + var tmp: Array[Byte] = IndexedErgoTreeSerializer.ergoTreeHash(box.ergoTree) + findModifierOpt(tmp) match { + case Some(n) => modifiers(n).asInstanceOf[IndexedErgoTree].addBox(bytesToId(box.id)) // ergotree found in last saveLimit blocks, update + case None => // ergotree not found in this TX + history.typedModifierById[IndexedErgoTree](bytesToId(tmp)) match { + case Some(x) => modifiers += x.addBox(bytesToId(box.id)) // ergotree found in DB, update + case None => modifiers += IndexedErgoTree(bytesToId(tmp), ListBuffer(bytesToId(box.id))) // ergotree not found at all, record } } globalBoxIndex += 1 - val addrHash: ModifierId = IndexedErgoAddressSerializer.addressToModifierId(IndexedErgoBoxSerializer.getAddress(box.ergoTree)) - boxesGroupedByAddress.put(addrHash, boxesGroupedByAddress.getOrElse[Seq[BoxId]](addrHash, emptyBoxSeq) :+ box.id) + tmp = IndexedErgoAddressSerializer.hashAddress(IndexedErgoBoxSerializer.getAddress(box.ergoTree)) + boxesGroupedByAddress.put(tmp, boxesGroupedByAddress.getOrElse[ListBuffer[BoxId]](tmp, emptyBoxBuffer) :+ box.id) }) //process boxes by address for(addressWithBoxes <- boxesGroupedByAddress) - updateMatch[IndexedErgoAddress](_.addressHash.eq(addressWithBoxes._1)) match { - case Some((iEa, n)) => modifiers.update(n, IndexedErgoAddress(addressWithBoxes._1, iEa.txIds :+ tx.id, iEa.boxIds ++ addressWithBoxes._2)) // address found in this block, update - case None => // address not found in this block - history.typedModifierById[IndexedErgoAddress](addressWithBoxes._1) match { - case Some(x) => modifiers += IndexedErgoAddress(addressWithBoxes._1, x.txIds :+ tx.id, x.boxIds ++ addressWithBoxes._2) //address found in DB, update - case None => modifiers += IndexedErgoAddress(addressWithBoxes._1, Seq(tx.id), addressWithBoxes._2) //address not found at all, record + findModifierOpt(addressWithBoxes._1) match { + case Some(n) => modifiers(n).asInstanceOf[IndexedErgoAddress].addTx(tx.id).addBoxes(addressWithBoxes._2) // address found in last saveLimit blocks, update + case None => // address not found in this block + history.typedModifierById[IndexedErgoAddress](bytesToId(addressWithBoxes._1)) match { + case Some(x) => modifiers += x.addTx(tx.id).addBoxes(addressWithBoxes._2) //address found in DB, update + case None => modifiers += IndexedErgoAddress(bytesToId(addressWithBoxes._1), ListBuffer(tx.id), addressWithBoxes._2) //address not found at all, record } } }) @@ -119,13 +125,7 @@ class ExtraIndex(chainSettings: ChainSettings) if(done) indexedHeight = height // update after caught up with chain - indexedHeightBuffer.clear() - globalTxIndexBuffer.clear() - globalBoxIndexBuffer.clear() - - historyStorage.insert(Seq((IndexedHeightKey , indexedHeightBuffer .putInt (indexedHeight ).array), - (GlobalTxIndexKey , globalTxIndexBuffer .putLong(globalTxIndex ).array), - (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), modifiers) + if(indexedHeight % saveLimit == 0 || done) saveProgress() // save index every saveLimit blocks or every block after caught up } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 8aa6f51711..872d1d69d5 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -12,9 +12,11 @@ import scorex.crypto.authds.ADKey import scorex.util.{ByteArrayOps, ModifierId, bytesToId, idToBytes} import scorex.util.serialization.{Reader, Writer} +import scala.collection.mutable.ListBuffer + case class IndexedErgoAddress(addressHash: ModifierId, - txIds: Seq[ModifierId], - boxIds: Seq[BoxId]) extends BlockSection { + txIds: ListBuffer[ModifierId], + boxIds: ListBuffer[BoxId]) extends BlockSection { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = idToBytes(addressHash) @@ -63,6 +65,16 @@ case class IndexedErgoAddress(addressHash: ModifierId, _utxos = _boxes.filter(!_.trackedBox.isSpent) this } + + def addTx(id: ModifierId): IndexedErgoAddress = { + txIds += id + this + } + + def addBoxes(boxes: ListBuffer[BoxId]): IndexedErgoAddress = { + boxIds ++= boxes + this + } } object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] { @@ -72,7 +84,9 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] withNetworkByte ++ hash256(withNetworkByte).take(ChecksumLength) } - def addressToModifierId(address: ErgoAddress): ModifierId = bytesToId(Algos.hash(addressToBytes(address))) + def hashAddress(address: ErgoAddress): Array[Byte] = Algos.hash(addressToBytes(address)) + + def addressToModifierId(address: ErgoAddress): ModifierId = bytesToId(hashAddress(address)) override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = { w.putBytes(idToBytes(iEa.addressHash)) @@ -85,11 +99,11 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] override def parse(r: Reader): IndexedErgoAddress = { val addressHash: ModifierId = bytesToId(r.getBytes(32)) val txnsLen: Long = r.getUInt() - var txns: Seq[ModifierId] = Seq.empty[ModifierId] - for(n <- 1L to txnsLen) txns = txns :+ r.getBytes(32).toModifierId + val txns: ListBuffer[ModifierId] = ListBuffer.empty[ModifierId] + for(n <- 1L to txnsLen) txns += r.getBytes(32).toModifierId val boxesLen: Long = r.getUInt() - var boxes: Seq[BoxId] = Seq.empty[BoxId] - for(n <- 1L to boxesLen) boxes = boxes :+ ADKey @@ r.getBytes(32) + val boxes: ListBuffer[BoxId] = ListBuffer.empty[BoxId] + for(n <- 1L to boxesLen) boxes += ADKey @@ r.getBytes(32) new IndexedErgoAddress(addressHash, txns, boxes) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala index c1a6b1deac..3065ee3ca1 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -12,8 +12,8 @@ import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.ErgoTree class IndexedErgoBox(val inclusionHeightOpt: Option[Int], - val spendingTxIdOpt: Option[ModifierId], - val spendingHeightOpt: Option[Int], + var spendingTxIdOpt: Option[ModifierId], + var spendingHeightOpt: Option[Int], val box: ErgoBox, val globalIndex: Long, val confirmations: Option[Int]) @@ -35,8 +35,11 @@ class IndexedErgoBox(val inclusionHeightOpt: Option[Int], def getAddress: ErgoAddress = IndexedErgoBoxSerializer.getAddress(box.ergoTree) - def asSpent(txId: ModifierId, txHeight: Int): IndexedErgoBox = - new IndexedErgoBox(inclusionHeightOpt, Some(txId), Some(txHeight), box, globalIndex, confirmations) + def asSpent(txId: ModifierId, txHeight: Int): IndexedErgoBox = { + spendingTxIdOpt = Some(txId) + spendingHeightOpt = Some(txHeight) + this + } } object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala index 1c0de445bf..09b46c0653 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala @@ -9,7 +9,9 @@ import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.ErgoTree -case class IndexedErgoTree(treeHash: ModifierId, boxIds: Seq[ModifierId]) extends BlockSection { +import scala.collection.mutable.ListBuffer + +case class IndexedErgoTree(treeHash: ModifierId, boxIds: ListBuffer[ModifierId]) extends BlockSection { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = idToBytes(treeHash) override def parentId: ModifierId = null @@ -24,6 +26,11 @@ case class IndexedErgoTree(treeHash: ModifierId, boxIds: Seq[ModifierId]) extend else boxIds ).map(history.typedModifierById[IndexedErgoBox](_).get) + + def addBox(id: ModifierId): IndexedErgoTree = { + boxIds += id + this + } } object IndexedErgoTreeSerializer extends ScorexSerializer[IndexedErgoTree] { @@ -39,8 +46,8 @@ object IndexedErgoTreeSerializer extends ScorexSerializer[IndexedErgoTree] { override def parse(r: Reader): IndexedErgoTree = { val treeHash: ModifierId = bytesToId(r.getBytes(32)) val boxIdsLen: Long = r.getUInt() - var boxIds: Seq[ModifierId] = Seq.empty[ModifierId] - for(n <- 1L to boxIdsLen) boxIds = boxIds :+ bytesToId(r.getBytes(32)) + val boxIds: ListBuffer[ModifierId] = ListBuffer.empty[ModifierId] + for(n <- 1L to boxIdsLen) boxIds += bytesToId(r.getBytes(32)) IndexedErgoTree(treeHash, boxIds) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 62fdf94472..6d41f00fb0 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -4,6 +4,7 @@ import com.google.common.cache.CacheBuilder import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.modifiers.history.HistoryModifierSerializer import org.ergoplatform.modifiers.history.header.Header +import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoBox, IndexedErgoTree} import org.ergoplatform.settings.{Algos, CacheSettings, ErgoSettings} import scorex.core.utils.ScorexEncoding import scorex.db.{ByteArrayWrapper, LDBFactory, LDBKVStore} @@ -32,21 +33,27 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c .maximumSize(config.history.blockSectionsCacheSize) .build[String, BlockSection] + private val extraCache = CacheBuilder.newBuilder() + .maximumSize(config.history.extraCacheSize) + .build[String, BlockSection] + private val indexCache = CacheBuilder.newBuilder() .maximumSize(config.history.indexesCacheSize) .build[ByteArrayWrapper, Array[Byte]] private def cacheModifier(mod: BlockSection): Unit = mod.modifierTypeId match { case Header.modifierTypeId => headersCache.put(mod.id, mod) + case IndexedErgoBox.modifierTypeId | IndexedErgoAddress.modifierTypeId | IndexedErgoTree.modifierTypeId => extraCache.put(mod.id, mod) case _ => blockSectionsCache.put(mod.id, mod) } private def lookupModifier(id: ModifierId): Option[BlockSection] = - Option(headersCache.getIfPresent(id)) orElse Option(blockSectionsCache.getIfPresent(id)) + Option(extraCache.getIfPresent(id)) orElse Option(headersCache.getIfPresent(id)) orElse Option(blockSectionsCache.getIfPresent(id)) private def removeModifier(id: ModifierId): Unit = { headersCache.invalidate(id) blockSectionsCache.invalidate(id) + extraCache.invalidate(id) } def modifierBytesById(id: ModifierId): Option[Array[Byte]] = { diff --git a/src/main/scala/org/ergoplatform/settings/CacheSettings.scala b/src/main/scala/org/ergoplatform/settings/CacheSettings.scala index 9977545d6b..22d62027eb 100644 --- a/src/main/scala/org/ergoplatform/settings/CacheSettings.scala +++ b/src/main/scala/org/ergoplatform/settings/CacheSettings.scala @@ -13,7 +13,7 @@ case class CacheSettings( mempool: MempoolCacheSettings ) -case class HistoryCacheSettings(blockSectionsCacheSize: Int, headersCacheSize: Int, indexesCacheSize: Int) +case class HistoryCacheSettings(blockSectionsCacheSize: Int, extraCacheSize: Int, headersCacheSize: Int, indexesCacheSize: Int) case class NetworkCacheSettings( invalidModifiersBloomFilterCapacity: Int, diff --git a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala index 67fa927a4a..1be0f66f3c 100644 --- a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala +++ b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala @@ -40,11 +40,12 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { rebroadcastCount = 3, minimalFeeAmount = 0, headerChainDiff = 100, - adProofsSuffixLength = 112*1024 + adProofsSuffixLength = 112*1024, + extraIndex = false ) settings.cacheSettings shouldBe CacheSettings( HistoryCacheSettings( - 12, 100, 1000 + 12, 250, 100, 1000 ), NetworkCacheSettings( invalidModifiersBloomFilterCapacity = 10000000, @@ -91,11 +92,12 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { rebroadcastCount = 3, minimalFeeAmount = 0, headerChainDiff = 100, - adProofsSuffixLength = 112*1024 + adProofsSuffixLength = 112*1024, + extraIndex = false ) settings.cacheSettings shouldBe CacheSettings( HistoryCacheSettings( - 12, 100, 1000 + 12, 250, 100, 1000 ), NetworkCacheSettings( invalidModifiersBloomFilterCapacity = 10000000, @@ -135,11 +137,12 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { rebroadcastCount = 3, minimalFeeAmount = 0, headerChainDiff = 100, - adProofsSuffixLength = 112*1024 + adProofsSuffixLength = 112*1024, + extraIndex = false ) settings.cacheSettings shouldBe CacheSettings( HistoryCacheSettings( - 12, 100, 1000 + 12, 250, 100, 1000 ), NetworkCacheSettings( invalidModifiersBloomFilterCapacity = 10000000, From 0cfc609b99b1175fb1f23ff60071b2d377e12066 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 21 Aug 2022 05:39:32 +0200 Subject: [PATCH 04/90] optimizations(cfor), time debug --- .../nodeView/history/extra/ExtraIndexer.scala | 112 +++++++++++------- .../history/extra/IndexedErgoAddress.scala | 25 ++-- .../history/extra/IndexedErgoTree.scala | 5 +- 3 files changed, 82 insertions(+), 60 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index d2ad9dc89c..0befe4f02e 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -1,21 +1,21 @@ package org.ergoplatform.nodeView.history.extra import akka.actor.{Actor, ActorRef, ActorSystem, Props} -import org.ergoplatform.ErgoBox.BoxId import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} -import org.ergoplatform.ErgoAddressEncoder +import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, Input} import org.ergoplatform.modifiers.history.BlockTransactions +import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start import org.ergoplatform.nodeView.history.storage.HistoryStorage -import org.ergoplatform.settings.{Algos, ChainSettings, ErgoAlgos} +import org.ergoplatform.settings.{Algos, ChainSettings} import scorex.db.ByteArrayWrapper import scorex.util.{ScorexLogging, bytesToId} import java.nio.ByteBuffer -import scala.collection.mutable import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import spire.syntax.all.cfor class ExtraIndex(chainSettings: ChainSettings) extends Actor with ScorexLogging { @@ -32,7 +32,7 @@ class ExtraIndex(chainSettings: ChainSettings) private var globalBoxIndex: Long = 0L private val globalBoxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) - private val saveLimit: Int = 10 + private val saveLimit: Int = 100 private var done: Boolean = false @@ -44,16 +44,23 @@ class ExtraIndex(chainSettings: ChainSettings) // fast access private val modifiers: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] - private val boxesGroupedByAddress: mutable.HashMap[Array[Byte], ListBuffer[BoxId]] = mutable.HashMap.empty[Array[Byte], ListBuffer[BoxId]] - private val emptyBoxBuffer: ListBuffer[BoxId] = ListBuffer.empty[BoxId] - private def findModifierOpt(id: Array[Byte]): Option[Int] = - modifiers.indexWhere(x => java.util.Arrays.equals(x.serializedId, id)) match { - case x: Int if x >= 0 => Some(x) - case _ => None + // timing + private var txs: (Long, Long) = (0L, 0L) + private var boxes: (Long, Long) = (0L, 0L) + private var write: (Long, Long) = (0L, 0L) + + private def findModifierOpt(id: Array[Byte]): Option[Int] = { + cfor(modifiers.size - 1)(_ >= 0, _ - 1) { i => // loop backwards == test latest modifiers first + if(java.util.Arrays.equals(modifiers(i).serializedId, id)) return Some(i) } + None + } private def saveProgress(): Unit = { + + write = write.copy(_1 = System.nanoTime()) + indexedHeightBuffer.clear() globalTxIndexBuffer.clear() globalBoxIndexBuffer.clear() @@ -62,66 +69,79 @@ class ExtraIndex(chainSettings: ChainSettings) (GlobalTxIndexKey , globalTxIndexBuffer .putLong(globalTxIndex ).array), (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), modifiers) + write = write.copy(_2 = System.nanoTime()) + + log.info(s"Wrote ${modifiers.size} extra indexes to database in ${write._2 - write._1}ns") + modifiers.clear() - boxesGroupedByAddress.clear() + } private def index(bt: BlockTransactions, height: Int): Unit = { //process transactions - bt.txIds.foreach(id => { - modifiers += IndexedErgoTransaction(bytesToId(id), indexedHeight, globalTxIndex) - modifiers += NumericTxIndex(globalTxIndex, bytesToId(id)) + txs = txs.copy(_1 = System.nanoTime()) + cfor(0)(_ < bt.txIds.size, _ + 1) { i => + modifiers += IndexedErgoTransaction(bytesToId(bt.txIds(i)), indexedHeight, globalTxIndex) + modifiers += NumericTxIndex(globalTxIndex, bytesToId(bt.txIds(i))) globalTxIndex += 1 - }) + } + txs = txs.copy(_2 = System.nanoTime()) + + boxes = boxes.copy(_1 = System.nanoTime()) + cfor(0)(_ < bt.txs.size, _ + 1) { n => - bt.txs.foreach(tx => { + val tx: ErgoTransaction = bt.txs(n) //process tx inputs if(indexedHeight != 1) { //only after 1st block (skip genesis box) - tx.inputs.foreach(in => - findModifierOpt(in.boxId) match { - case Some(n) => modifiers(n).asInstanceOf[IndexedErgoBox].asSpent(tx.id, indexedHeight) // box found in last saveLimit blocks, update - case None => // box not found in this block - history.typedModifierById[IndexedErgoBox](bytesToId(in.boxId)) match { + cfor(0)(_ < tx.inputs.size, _ + 1) { i => + val input: Input = tx.inputs(i) + findModifierOpt(input.boxId) match { + case Some(x) => modifiers(x).asInstanceOf[IndexedErgoBox].asSpent(tx.id, indexedHeight) // box found in last saveLimit blocks, update + case None => // box not found in last saveLimit blocks + history.typedModifierById[IndexedErgoBox](bytesToId(input.boxId)) match { case Some(x) => modifiers += x.asSpent(tx.id, indexedHeight) // box found in DB, update - case None => log.warn(s"Input for box ${ErgoAlgos.encode(in.boxId)} not found in database") // box not found at all (this shouldn't happen) + case None => log.warn(s"Input for box ${bytesToId(input.boxId)} not found in database") // box not found at all (this shouldn't happen) } } - ) + } } //process tx outputs - tx.outputs.foreach(box => { + cfor(0)(_ < tx.outputs.size, _ + 1) { i => + val box: ErgoBox = tx.outputs(i) modifiers += new IndexedErgoBox(Some(indexedHeight), None, None, box, globalBoxIndex, Some(chainHeight - indexedHeight)) // box by id modifiers += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number - var tmp: Array[Byte] = IndexedErgoTreeSerializer.ergoTreeHash(box.ergoTree) - findModifierOpt(tmp) match { - case Some(n) => modifiers(n).asInstanceOf[IndexedErgoTree].addBox(bytesToId(box.id)) // ergotree found in last saveLimit blocks, update - case None => // ergotree not found in this TX - history.typedModifierById[IndexedErgoTree](bytesToId(tmp)) match { + globalBoxIndex += 1 + + // box by ergotree + val treeHash: Array[Byte] = IndexedErgoTreeSerializer.ergoTreeHash(box.ergoTree) + findModifierOpt(treeHash) match { + case Some(x) => modifiers(x).asInstanceOf[IndexedErgoTree].addBox(bytesToId(box.id)) // ergotree found in last saveLimit blocks, update + case None => // ergotree not found in last saveLimit blocks + history.typedModifierById[IndexedErgoTree](bytesToId(treeHash)) match { case Some(x) => modifiers += x.addBox(bytesToId(box.id)) // ergotree found in DB, update - case None => modifiers += IndexedErgoTree(bytesToId(tmp), ListBuffer(bytesToId(box.id))) // ergotree not found at all, record + case None => modifiers += IndexedErgoTree(bytesToId(treeHash), ListBuffer(bytesToId(box.id))) // ergotree not found at all, record } } - globalBoxIndex += 1 - tmp = IndexedErgoAddressSerializer.hashAddress(IndexedErgoBoxSerializer.getAddress(box.ergoTree)) - boxesGroupedByAddress.put(tmp, boxesGroupedByAddress.getOrElse[ListBuffer[BoxId]](tmp, emptyBoxBuffer) :+ box.id) - }) - - //process boxes by address - for(addressWithBoxes <- boxesGroupedByAddress) - findModifierOpt(addressWithBoxes._1) match { - case Some(n) => modifiers(n).asInstanceOf[IndexedErgoAddress].addTx(tx.id).addBoxes(addressWithBoxes._2) // address found in last saveLimit blocks, update - case None => // address not found in this block - history.typedModifierById[IndexedErgoAddress](bytesToId(addressWithBoxes._1)) match { - case Some(x) => modifiers += x.addTx(tx.id).addBoxes(addressWithBoxes._2) //address found in DB, update - case None => modifiers += IndexedErgoAddress(bytesToId(addressWithBoxes._1), ListBuffer(tx.id), addressWithBoxes._2) //address not found at all, record + + // box by address + val addrHash: Array[Byte] = IndexedErgoAddressSerializer.hashAddress(IndexedErgoBoxSerializer.getAddress(box.ergoTree)) + findModifierOpt(addrHash) match { + case Some(x) => modifiers(x).asInstanceOf[IndexedErgoAddress].addBox(box.id) // address found in last saveLimit blocks, update + case None => // address not found in last saveLimit blocks + history.typedModifierById[IndexedErgoAddress](bytesToId(addrHash)) match { + case Some(x) => modifiers += x.addTx(tx.id).addBox(box.id)//address found in DB, update + case None => modifiers += IndexedErgoAddress(bytesToId(addrHash), ListBuffer(tx.id), ListBuffer(box.id)) //address not found at all, record } } - }) + } + + } + boxes = boxes.copy(_2 = System.nanoTime()) - log.info(s"Indexed block #$indexedHeight [transactions: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] - progress: $indexedHeight / $chainHeight") + log.info(s"Indexed block #$indexedHeight (transactions: ${bt.txs.size} [${txs._2 - txs._1}ns], boxes: ${bt.txs.map(_.outputs.size).sum} [${boxes._2 - boxes._1}ns]) - progress: $indexedHeight / $chainHeight (mods: ${modifiers.size})") if(done) indexedHeight = height // update after caught up with chain diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 872d1d69d5..05f7789e26 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -13,6 +13,7 @@ import scorex.util.{ByteArrayOps, ModifierId, bytesToId, idToBytes} import scorex.util.serialization.{Reader, Writer} import scala.collection.mutable.ListBuffer +import spire.syntax.all.cfor case class IndexedErgoAddress(addressHash: ModifierId, txIds: ListBuffer[ModifierId], @@ -25,19 +26,19 @@ case class IndexedErgoAddress(addressHash: ModifierId, override type M = IndexedErgoAddress override def serializer: ScorexSerializer[IndexedErgoAddress] = IndexedErgoAddressSerializer - private var _transactions: Seq[IndexedErgoTransaction] = Seq.empty[IndexedErgoTransaction] - private var _boxes: Seq[IndexedErgoBox] = Seq.empty[IndexedErgoBox] - private var _utxos: Seq[IndexedErgoBox] = _ + private var _transactions: ListBuffer[IndexedErgoTransaction] = ListBuffer.empty[IndexedErgoTransaction] + private var _boxes: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] + private var _utxos: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] - def transactions(lastN: Long = 20L): Seq[IndexedErgoTransaction] = + def transactions(lastN: Long = 20L): ListBuffer[IndexedErgoTransaction] = if(lastN > 0) _transactions.slice(math.max((_transactions.size - lastN).toInt, 0), _transactions.size) else _transactions - def boxes(): Seq[IndexedErgoBox] = _boxes + def boxes(): ListBuffer[IndexedErgoBox] = _boxes - def utxos(): Seq[IndexedErgoBox] = _utxos + def utxos(): ListBuffer[IndexedErgoBox] = _utxos def retrieveBody(history: ErgoHistoryReader): IndexedErgoAddress = { retrieveTxs(history, -1) @@ -71,8 +72,8 @@ case class IndexedErgoAddress(addressHash: ModifierId, this } - def addBoxes(boxes: ListBuffer[BoxId]): IndexedErgoAddress = { - boxIds ++= boxes + def addBox(box: BoxId): IndexedErgoAddress = { + boxIds += box this } } @@ -91,19 +92,19 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = { w.putBytes(idToBytes(iEa.addressHash)) w.putUInt(iEa.txIds.length) - iEa.txIds.foreach(m => w.putBytes(m.toBytes)) + cfor(0)(_ < iEa.txIds.length, _ + 1) { i => w.putBytes(iEa.txIds(i).toBytes)} w.putUInt(iEa.boxIds.length) - iEa.boxIds.foreach(w.putBytes(_)) + cfor(0)(_ < iEa.boxIds.length, _ + 1) { i => w.putBytes(iEa.boxIds(i))} } override def parse(r: Reader): IndexedErgoAddress = { val addressHash: ModifierId = bytesToId(r.getBytes(32)) val txnsLen: Long = r.getUInt() val txns: ListBuffer[ModifierId] = ListBuffer.empty[ModifierId] - for(n <- 1L to txnsLen) txns += r.getBytes(32).toModifierId + cfor(0)(_ < txnsLen, _ + 1) {_ => txns += r.getBytes(32).toModifierId} val boxesLen: Long = r.getUInt() val boxes: ListBuffer[BoxId] = ListBuffer.empty[BoxId] - for(n <- 1L to boxesLen) boxes += ADKey @@ r.getBytes(32) + cfor(0)(_ < boxesLen, _ + 1) { _ => boxes += ADKey @@ r.getBytes(32)} new IndexedErgoAddress(addressHash, txns, boxes) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala index 09b46c0653..40555257cf 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala @@ -10,6 +10,7 @@ import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.ErgoTree import scala.collection.mutable.ListBuffer +import spire.syntax.all.cfor case class IndexedErgoTree(treeHash: ModifierId, boxIds: ListBuffer[ModifierId]) extends BlockSection { override val sizeOpt: Option[Int] = None @@ -40,14 +41,14 @@ object IndexedErgoTreeSerializer extends ScorexSerializer[IndexedErgoTree] { override def serialize(iEt: IndexedErgoTree, w: Writer): Unit = { w.putBytes(iEt.serializedId) w.putUInt(iEt.boxIds.length) - iEt.boxIds.foreach(id => w.putBytes(idToBytes(id))) + cfor(0)(_ < iEt.boxIds.length, _ + 1) { i => w.putBytes(idToBytes(iEt.boxIds(i)))} } override def parse(r: Reader): IndexedErgoTree = { val treeHash: ModifierId = bytesToId(r.getBytes(32)) val boxIdsLen: Long = r.getUInt() val boxIds: ListBuffer[ModifierId] = ListBuffer.empty[ModifierId] - for(n <- 1L to boxIdsLen) boxIds += bytesToId(r.getBytes(32)) + cfor(0)(_ < boxIdsLen, _ + 1) { _ => boxIds += bytesToId(r.getBytes(32))} IndexedErgoTree(treeHash, boxIds) } } From 53d6909f0186541a6e9444a5f69c0fcd8e297196 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 21 Aug 2022 16:43:09 +0200 Subject: [PATCH 05/90] changed write interval --- src/main/scala/org/ergoplatform/ErgoApp.scala | 2 +- .../nodeView/history/extra/ExtraIndexer.scala | 39 +++++++------------ 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index 9b5eb0261f..43a5139e39 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -104,7 +104,7 @@ class ErgoApp(args: Args) extends ScorexLogging { // Create an instance of ExtraIndexer actor if "extraIndex = true" in config private val indexerRefOpt: Option[ActorRef] = if(ergoSettings.nodeSettings.extraIndex) - Some(ExtraIndexerRef(ergoSettings.chainSettings)) + Some(ExtraIndexerRef(ergoSettings.chainSettings, ergoSettings.cacheSettings)) else None ExtraIndexerRef.setAddressEncoder(ergoSettings.addressEncoder) // initialize an accessible address encoder regardless of extra indexing being enabled diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 0befe4f02e..80133ae4c3 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -9,7 +9,7 @@ import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.Sema import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start import org.ergoplatform.nodeView.history.storage.HistoryStorage -import org.ergoplatform.settings.{Algos, ChainSettings} +import org.ergoplatform.settings.{Algos, CacheSettings, ChainSettings} import scorex.db.ByteArrayWrapper import scorex.util.{ScorexLogging, bytesToId} @@ -17,7 +17,7 @@ import java.nio.ByteBuffer import scala.collection.mutable.{ArrayBuffer, ListBuffer} import spire.syntax.all.cfor -class ExtraIndex(chainSettings: ChainSettings) +class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) extends Actor with ScorexLogging { private val IndexedHeightKey: ByteArrayWrapper = ByteArrayWrapper.apply(Algos.hash("indexed height")) @@ -32,7 +32,7 @@ class ExtraIndex(chainSettings: ChainSettings) private var globalBoxIndex: Long = 0L private val globalBoxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) - private val saveLimit: Int = 100 + private val saveLimit: Int = cacheSettings.history.extraCacheSize * 25 private var done: Boolean = false @@ -45,13 +45,8 @@ class ExtraIndex(chainSettings: ChainSettings) // fast access private val modifiers: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] - // timing - private var txs: (Long, Long) = (0L, 0L) - private var boxes: (Long, Long) = (0L, 0L) - private var write: (Long, Long) = (0L, 0L) - private def findModifierOpt(id: Array[Byte]): Option[Int] = { - cfor(modifiers.size - 1)(_ >= 0, _ - 1) { i => // loop backwards == test latest modifiers first + cfor(modifiers.size - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first if(java.util.Arrays.equals(modifiers(i).serializedId, id)) return Some(i) } None @@ -59,7 +54,7 @@ class ExtraIndex(chainSettings: ChainSettings) private def saveProgress(): Unit = { - write = write.copy(_1 = System.nanoTime()) + val start: Long = System.nanoTime() indexedHeightBuffer.clear() globalTxIndexBuffer.clear() @@ -69,9 +64,9 @@ class ExtraIndex(chainSettings: ChainSettings) (GlobalTxIndexKey , globalTxIndexBuffer .putLong(globalTxIndex ).array), (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), modifiers) - write = write.copy(_2 = System.nanoTime()) + val end: Long = System.nanoTime() - log.info(s"Wrote ${modifiers.size} extra indexes to database in ${write._2 - write._1}ns") + log.info(s"Wrote ${modifiers.size} extra indexes to database in ${(end - start) / 1000000000D}s") modifiers.clear() @@ -80,15 +75,12 @@ class ExtraIndex(chainSettings: ChainSettings) private def index(bt: BlockTransactions, height: Int): Unit = { //process transactions - txs = txs.copy(_1 = System.nanoTime()) cfor(0)(_ < bt.txIds.size, _ + 1) { i => modifiers += IndexedErgoTransaction(bytesToId(bt.txIds(i)), indexedHeight, globalTxIndex) modifiers += NumericTxIndex(globalTxIndex, bytesToId(bt.txIds(i))) globalTxIndex += 1 } - txs = txs.copy(_2 = System.nanoTime()) - boxes = boxes.copy(_1 = System.nanoTime()) cfor(0)(_ < bt.txs.size, _ + 1) { n => val tx: ErgoTransaction = bt.txs(n) @@ -98,7 +90,7 @@ class ExtraIndex(chainSettings: ChainSettings) cfor(0)(_ < tx.inputs.size, _ + 1) { i => val input: Input = tx.inputs(i) findModifierOpt(input.boxId) match { - case Some(x) => modifiers(x).asInstanceOf[IndexedErgoBox].asSpent(tx.id, indexedHeight) // box found in last saveLimit blocks, update + case Some(x) => modifiers(x).asInstanceOf[IndexedErgoBox].asSpent(tx.id, indexedHeight) // box found in last saveLimit modifiers, update case None => // box not found in last saveLimit blocks history.typedModifierById[IndexedErgoBox](bytesToId(input.boxId)) match { case Some(x) => modifiers += x.asSpent(tx.id, indexedHeight) // box found in DB, update @@ -118,7 +110,7 @@ class ExtraIndex(chainSettings: ChainSettings) // box by ergotree val treeHash: Array[Byte] = IndexedErgoTreeSerializer.ergoTreeHash(box.ergoTree) findModifierOpt(treeHash) match { - case Some(x) => modifiers(x).asInstanceOf[IndexedErgoTree].addBox(bytesToId(box.id)) // ergotree found in last saveLimit blocks, update + case Some(x) => modifiers(x).asInstanceOf[IndexedErgoTree].addBox(bytesToId(box.id)) // ergotree found in last saveLimit modifiers, update case None => // ergotree not found in last saveLimit blocks history.typedModifierById[IndexedErgoTree](bytesToId(treeHash)) match { case Some(x) => modifiers += x.addBox(bytesToId(box.id)) // ergotree found in DB, update @@ -129,7 +121,7 @@ class ExtraIndex(chainSettings: ChainSettings) // box by address val addrHash: Array[Byte] = IndexedErgoAddressSerializer.hashAddress(IndexedErgoBoxSerializer.getAddress(box.ergoTree)) findModifierOpt(addrHash) match { - case Some(x) => modifiers(x).asInstanceOf[IndexedErgoAddress].addBox(box.id) // address found in last saveLimit blocks, update + case Some(x) => modifiers(x).asInstanceOf[IndexedErgoAddress].addBox(box.id) // address found in last saveLimit modifiers, update case None => // address not found in last saveLimit blocks history.typedModifierById[IndexedErgoAddress](bytesToId(addrHash)) match { case Some(x) => modifiers += x.addTx(tx.id).addBox(box.id)//address found in DB, update @@ -139,13 +131,12 @@ class ExtraIndex(chainSettings: ChainSettings) } } - boxes = boxes.copy(_2 = System.nanoTime()) - log.info(s"Indexed block #$indexedHeight (transactions: ${bt.txs.size} [${txs._2 - txs._1}ns], boxes: ${bt.txs.map(_.outputs.size).sum} [${boxes._2 - boxes._1}ns]) - progress: $indexedHeight / $chainHeight (mods: ${modifiers.size})") + log.info(s"Indexed block #$indexedHeight [transactions: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] - progress: $indexedHeight / $chainHeight (mods: ${modifiers.size})") - if(done) indexedHeight = height // update after caught up with chain + if(done) indexedHeight = height // update height here after caught up with chain - if(indexedHeight % saveLimit == 0 || done) saveProgress() // save index every saveLimit blocks or every block after caught up + if(modifiers.size >= saveLimit || done) saveProgress() // save index every saveLimit modifiers or every block after caught up } @@ -197,8 +188,8 @@ object ExtraIndexerRef { def setAddressEncoder(encoder: ErgoAddressEncoder): Unit = if(_ae == null) _ae = encoder - def apply(chainSettings: ChainSettings)(implicit system: ActorSystem): ActorRef = { - val actor = system.actorOf(Props.create(classOf[ExtraIndex], chainSettings)) + def apply(chainSettings: ChainSettings, cacheSettings: CacheSettings)(implicit system: ActorSystem): ActorRef = { + val actor = system.actorOf(Props.create(classOf[ExtraIndex], chainSettings, cacheSettings)) _ae = chainSettings.addressEncoder ExtraIndexerRefHolder.init(actor) actor From 80bbba0b20cb75952d09187a95a233cf18b39331 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 21 Aug 2022 17:33:52 +0200 Subject: [PATCH 06/90] changed to manual serialization and insertion --- .../nodeView/history/extra/ExtraIndexer.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 80133ae4c3..95a1684ce6 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -3,7 +3,7 @@ package org.ergoplatform.nodeView.history.extra import akka.actor.{Actor, ActorRef, ActorSystem, Props} import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, Input} -import org.ergoplatform.modifiers.history.BlockTransactions +import org.ergoplatform.modifiers.history.{BlockTransactions, HistoryModifierSerializer} import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} @@ -56,13 +56,17 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) val start: Long = System.nanoTime() + cfor(0)(_ < modifiers.length, _ + 1) { i => + historyStorage.insert(modifiers(i).serializedId, HistoryModifierSerializer.toBytes(modifiers(i))) + } + indexedHeightBuffer.clear() globalTxIndexBuffer.clear() globalBoxIndexBuffer.clear() historyStorage.insert(Seq((IndexedHeightKey , indexedHeightBuffer .putInt (indexedHeight ).array), (GlobalTxIndexKey , globalTxIndexBuffer .putLong(globalTxIndex ).array), - (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), modifiers) + (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), Seq.empty) val end: Long = System.nanoTime() From 0d4be4d0e69cb0437cc335f315f29f68a4a79dfb Mon Sep 17 00:00:00 2001 From: jellymlg Date: Tue, 23 Aug 2022 17:40:31 +0200 Subject: [PATCH 07/90] optimized db operations, fixed modifier size getting too big --- .gitignore | 1 + avldb/build.sbt | 3 +- .../src/main/scala/scorex/db/LDBKVStore.scala | 11 +- src/main/resources/api/openapi.yaml | 120 ++++++++++----- src/main/resources/application.conf | 2 +- src/main/scala/org/ergoplatform/ErgoApp.scala | 2 +- .../org/ergoplatform/http/api/ApiCodecs.scala | 11 +- ...piRoute.scala => BlockchainApiRoute.scala} | 120 ++++++++------- .../network/ErgoNodeViewSynchronizer.scala | 5 +- .../nodeView/history/ErgoHistory.scala | 22 +-- .../nodeView/history/extra/ExtraIndexer.scala | 75 ++++++---- .../history/extra/IndexedErgoAddress.scala | 138 +++++++++++------- .../history/extra/IndexedErgoTree.scala | 76 ++++++++-- .../nodeView/history/extra/NumericIndex.scala | 15 +- .../history/storage/HistoryStorage.scala | 41 +++--- .../FullBlockProcessor.scala | 13 +- .../FullBlockSectionProcessor.scala | 5 +- .../modifierprocessors/HeadersProcessor.scala | 6 +- .../wallet/persistence/WalletStorage.scala | 12 +- .../settings/ErgoSettingsSpecification.scala | 6 +- 20 files changed, 412 insertions(+), 272 deletions(-) rename src/main/scala/org/ergoplatform/http/api/{ExtraIndexApiRoute.scala => BlockchainApiRoute.scala} (56%) diff --git a/.gitignore b/.gitignore index 1deb9ac892..626004c1c7 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ local.conf .metals/ .vscode/ project/metals.sbt +null/ # scala worksheets *.worksheet.sc \ No newline at end of file diff --git a/avldb/build.sbt b/avldb/build.sbt index 7d3e95dba6..3378dd64ee 100644 --- a/avldb/build.sbt +++ b/avldb/build.sbt @@ -16,7 +16,8 @@ libraryDependencies ++= Seq( ) libraryDependencies ++= Seq( - "org.ethereum" % "leveldbjni-all" % "1.18.3" + "org.ethereum" % "leveldbjni-all" % "1.18.3", + "org.typelevel" %% "spire" % "0.14.1" ) testOptions in Test := Seq(Tests.Filter(t => !t.matches(".*Benchmark$"))) diff --git a/avldb/src/main/scala/scorex/db/LDBKVStore.scala b/avldb/src/main/scala/scorex/db/LDBKVStore.scala index bdf3ade613..edbc958a7e 100644 --- a/avldb/src/main/scala/scorex/db/LDBKVStore.scala +++ b/avldb/src/main/scala/scorex/db/LDBKVStore.scala @@ -4,6 +4,7 @@ import org.iq80.leveldb.DB import scorex.util.ScorexLogging import scala.util.{Failure, Success, Try} +import spire.syntax.all.cfor /** @@ -13,11 +14,11 @@ import scala.util.{Failure, Success, Try} */ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging { - def update(toInsert: Seq[(K, V)], toRemove: Seq[K]): Try[Unit] = { + def update(toInsert: Array[(K, V)], toRemove: Array[K]): Try[Unit] = { val batch = db.createWriteBatch() try { - toInsert.foreach { case (k, v) => batch.put(k, v) } - toRemove.foreach(batch.delete) + cfor(0)(_ < toInsert.length, _ + 1) { i => batch.put(toInsert(i)._1, toInsert(i)._2)} + cfor(0)(_ < toRemove.length, _ + 1) { i => batch.delete(toRemove(i))} db.write(batch) Success(()) } catch { @@ -42,9 +43,9 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging } } - def insert(values: Seq[(K, V)]): Try[Unit] = update(values, Seq.empty) + def insert(values: Array[(K, V)]): Try[Unit] = update(values, Array.empty) - def remove(keys: Seq[K]): Try[Unit] = update(Seq.empty, keys) + def remove(keys: Array[K]): Try[Unit] = update(Array.empty, keys) /** * Get last key within some range (inclusive) by used comparator. diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 2892a53c10..6ffb0119d1 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -5191,12 +5191,12 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/transaction/byId/{txId}: + /blockchain/transaction/byId/{txId}: get: summary: Retrieve a transaction by its id operationId: getTxById tags: - - extra + - blockchain parameters: - in: path name: txId @@ -5224,12 +5224,12 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/transaction/byIndex/{txIndex}: + /blockchain/transaction/byIndex/{txIndex}: get: summary: Retrieve a transaction by global index number operationId: getTxByIndex tags: - - extra + - blockchain parameters: - in: path name: txIndex @@ -5257,12 +5257,12 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/transaction/byAddress/{address}: + /blockchain/transaction/byAddress/{address}: get: summary: Retrieve transactions by their associated address operationId: getTxsByAddress tags: - - extra + - blockchain parameters: - in: path name: address @@ -5270,14 +5270,22 @@ paths: description: adderess associated with transactions schema: $ref: '#/components/schemas/ErgoAddress' + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int64 + default: 0 - in: query name: limit required: false - description: get last n elements, set to negative for all elements + description: amount of elements to retrieve schema: type: integer format: int64 - default: 20 + default: 10 responses: '200': description: transactions associated with wanted address @@ -5301,25 +5309,25 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/transaction/range: + /blockchain/transaction/range: get: summary: Get a range of transaction ids operationId: getTxRange tags: - - extra + - blockchain parameters: - in: query - name: fromIndex + name: offset required: false - description: Min transaction index + description: amount of elements to skip from the start schema: type: integer format: int64 default: 0 - in: query - name: toIndex + name: limit required: false - description: Max transaction index + description: amount of elements to retrieve schema: type: integer format: int64 @@ -5341,12 +5349,12 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/box/byId/{boxId}: + /blockchain/box/byId/{boxId}: get: summary: Retrieve a box by its id operationId: getBoxById tags: - - extra + - blockchain parameters: - in: path name: boxId @@ -5374,12 +5382,12 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/box/byIndex/{boxIndex}: + /blockchain/box/byIndex/{boxIndex}: get: summary: Retrieve a box by global index number operationId: getBoxByIndex tags: - - extra + - blockchain parameters: - in: path name: boxIndex @@ -5407,12 +5415,12 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/box/byAddress/{address}: + /blockchain/box/byAddress/{address}: get: summary: Retrieve boxes by their associated address operationId: getBoxesByAddress tags: - - extra + - blockchain parameters: - in: path name: address @@ -5420,14 +5428,22 @@ paths: description: adderess associated with boxes schema: $ref: '#/components/schemas/ErgoAddress' + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int64 + default: 0 - in: query name: limit required: false - description: get last n elements, set to negative for all elements + description: amount of elements to retrieve schema: type: integer format: int64 - default: 20 + default: 10 responses: '200': description: boxes associated with wanted address @@ -5451,12 +5467,12 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/box/unspent/byAddress/{address}: + /blockchain/box/unspent/byAddress/{address}: get: summary: Retrieve unspent boxes by their associated address operationId: getBoxesByAddressUnspent tags: - - extra + - blockchain parameters: - in: path name: address @@ -5464,14 +5480,22 @@ paths: description: adderess associated with unspent boxes schema: $ref: '#/components/schemas/ErgoAddress' + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int64 + default: 0 - in: query name: limit required: false - description: get last n elements, set to negative for all elements + description: amount of elements to retrieve schema: type: integer format: int64 - default: 20 + default: 10 responses: '200': description: unspent boxes associated with wanted address @@ -5495,25 +5519,25 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/box/range: + /blockchain/box/range: get: summary: Get a range of box ids operationId: getBoxRange tags: - - extra + - blockchain parameters: - in: query - name: fromIndex + name: offset required: false - description: Min box index + description: amount of elements to skip from the start schema: type: integer format: int64 default: 0 - in: query - name: toIndex + name: limit required: false - description: Max box index + description: amount of elements to retrieve schema: type: integer format: int64 @@ -5535,12 +5559,12 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/box/byErgoTree/{ergoTreeHex}: + /blockchain/box/byErgoTree/{ergoTreeHex}: get: summary: Retrieve boxes by their associated ergotree operationId: getBoxesByErgoTree tags: - - extra + - blockchain parameters: - in: path name: ergoTreeHex @@ -5549,14 +5573,22 @@ paths: schema: type: string example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int64 + default: 0 - in: query name: limit required: false - description: get last n elements, set to negative for all elements + description: amount of elements to retrieve schema: type: integer format: int64 - default: 20 + default: 10 responses: '200': description: boxes with wanted ergotree @@ -5577,12 +5609,12 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /extra/box/unspent/byErgoTree/{ergoTreeHex}: + /blockchain/box/unspent/byErgoTree/{ergoTreeHex}: get: summary: Retrieve unspent boxes by their associated ergotree operationId: getBoxesByErgoTreeUnspent tags: - - extra + - blockchain parameters: - in: path name: ergoTreeHex @@ -5591,14 +5623,22 @@ paths: schema: type: string example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' + - in: query + name: offset + required: false + description: amount of elements to skip from the start + schema: + type: integer + format: int64 + default: 0 - in: query name: limit required: false - description: get last n elements, set to negative for all elements + description: amount of elements to retrieve schema: type: integer format: int64 - default: 20 + default: 10 responses: '200': description: unspent boxes with wanted ergotree diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 5b775f5bab..aa964b1e24 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -111,7 +111,7 @@ ergo { blockSectionsCacheSize = 12 # Number of recently used extra indexes that will be kept in memory (has no effect if extraIndex is disabled) - extraCacheSize = 250 + extraCacheSize = 500 # Number of recently used headers that will be kept in memory headersCacheSize = 1000 diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index 43a5139e39..23b11c2537 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -132,7 +132,7 @@ class ErgoApp(args: Args) extends ScorexLogging { private val apiRoutes: Seq[ApiRoute] = Seq( EmissionApiRoute(ergoSettings), ErgoUtilsApiRoute(ergoSettings), - ExtraIndexApiRoute(readersHolderRef, ergoSettings, indexerRefOpt), + BlockchainApiRoute(readersHolderRef, ergoSettings, indexerRefOpt), ErgoPeersApiRoute(peerManagerRef, networkControllerRef, syncTracker, deliveryTracker, scorexSettings.restApi), InfoApiRoute(statsCollectorRef, scorexSettings.restApi, timeProvider), BlocksApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings), diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index 981033f2f4..c6724e4a41 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -28,7 +28,7 @@ import sigmastate.interpreter._ import sigmastate.interpreter.CryptoConstants.EcPointType import io.circe.syntax._ import org.ergoplatform.http.api.requests.{CryptoResult, ExecuteRequest, HintExtractionRequest} -import org.ergoplatform.nodeView.history.extra.{ExtraIndexerRef, IndexedErgoAddress, IndexedErgoBox, IndexedErgoTransaction} +import org.ergoplatform.nodeView.history.extra.{ExtraIndexerRef, IndexedErgoBox, IndexedErgoTransaction} import org.ergoplatform.wallet.interface4j.SecretString import scorex.crypto.authds.{LeafData, Side} import scorex.crypto.authds.merkle.MerkleProof @@ -493,15 +493,6 @@ trait ApiCodecs extends JsonCodecs { "size" -> tx.txSize.asJson) } - implicit val indexedAddressEncoder: Encoder[IndexedErgoAddress] = { address => - Json.obj( - "transactions" -> address.transactions().map(_.asJson).asJson, - "txCount" -> address.transactions().size.asJson, - "utxos" -> address.utxos().asJson, - "balance" -> address.utxos().map(_.box.value).sum.asJson, - "tokens" -> address.utxos().map(_.box.additionalTokens.toArray.toSeq.asJson).asJson - ) - } } trait ApiEncoderOption diff --git a/src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala similarity index 56% rename from src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala rename to src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 0d167cf562..cf863daeff 100644 --- a/src/main/scala/org/ergoplatform/http/api/ExtraIndexApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -19,21 +19,19 @@ import sigmastate.serialization.ErgoTreeSerializer import scala.concurrent.Future import scala.util.{Failure, Success} -case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings, extraIndexerOpt: Option[ActorRef]) +case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings, extraIndexerOpt: Option[ActorRef]) (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute with ApiCodecs { val settings: RESTApiSettings = ergoSettings.scorexSettings.restApi - val paging: Directive[(Long, Long)] = parameters("fromIndex".as[Long] ? 0L, "toIndex".as[Long] ? 10L) + val paging: Directive[(Long, Long)] = parameters("offset".as[Long] ? 0L, "limit".as[Long] ? 10L) - val lastN: Directive[Tuple1[Long]] = parameter("limit".as[Long] ? 20L) + private val MaxItems = IndexedErgoAddress.segmentTreshold val addressEncoder: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder - private val MaxTxs = 16384 - private val MaxBoxes = MaxTxs * 3 - override val route: Route = pathPrefix("extra") { + override val route: Route = pathPrefix("blockchain") { getTxByIdR ~ getTxByIndexR ~ getTxsByAddressR ~ @@ -79,35 +77,36 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting ApiResponse(getTxByIndex(index)) } - private def getTxsByAddress(addr: ErgoAddress, lastN: Long): Future[Option[Seq[IndexedErgoTransaction]]] = + private def getTxsByAddress(addr: ErgoAddress, offset: Long, limit: Long): Future[Option[Seq[IndexedErgoTransaction]]] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => Some(addr.retrieveTxs(history, lastN).transactions(-1)) + case Some(addr) => Some(addr.retrieveTxs(history, offset, limit)) case None => None } } - private def getTxsByAddressR: Route = (get & pathPrefix("transaction" / "byAddress") & path(Segment) & lastN) { (address, limit) => - addressEncoder.fromString(address) match { - case Success(addr) => ApiResponse(getTxsByAddress(addr, limit)) - case Failure(_) => BadRequest("Incorrect address format") + private def getTxsByAddressR: Route = (get & pathPrefix("transaction" / "byAddress") & path(Segment) & paging) { (address, offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems transactions can be requested") + }else { + addressEncoder.fromString(address) match { + case Success(addr) => ApiResponse(getTxsByAddress(addr, offset, limit)) + case Failure(_) => BadRequest("Incorrect address format") + } } } - private def getTxRange(fromHeight: Long, toHeight: Long): Future[Seq[ModifierId]] = + private def getTxRange(offset: Long, limit: Long): Future[Seq[ModifierId]] = getHistory.map { history => - (for(n <- fromHeight to toHeight) - yield history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(n))).get.m - ).toSeq + val base: Int = (history.fullBlockHeight - offset).toInt + for(n <- (base - limit) to base) yield history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(n))).get.m } - private def getTxRangeR: Route = (pathPrefix("transaction" / "range") & paging) { (fromIndex, toIndex) => - if (toIndex < fromIndex) { - BadRequest("toIndex < fromIndex") - } else if (fromIndex - toIndex > MaxTxs) { - BadRequest(s"No more than $MaxTxs transactions can be requested") - } else { - ApiResponse(getTxRange(fromIndex, toIndex)) + private def getTxRangeR: Route = (pathPrefix("transaction" / "range") & paging) { (offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems transactions can be requested") + }else { + ApiResponse(getTxRange(offset, limit)) } } @@ -132,80 +131,93 @@ case class ExtraIndexApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting ApiResponse(getBoxByIndex(index)) } - private def getBoxesByAddress(addr: ErgoAddress, lastN: Long): Future[Seq[IndexedErgoBox]] = + private def getBoxesByAddress(addr: ErgoAddress, offset: Long, limit: Long): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => addr.retrieveBoxes(history, lastN).boxes() + case Some(addr) => addr.retrieveBoxes(history, offset, limit) case None => Seq.empty[IndexedErgoBox] } } - private def getBoxesByAddressR: Route = (get & pathPrefix("box" / "byAddress") & path(Segment) & lastN) { (address, limit) => - addressEncoder.fromString(address) match { - case Success(addr) => ApiResponse(getBoxesByAddress(addr, limit)) - case Failure(_) => BadRequest("Incorrect address format") + private def getBoxesByAddressR: Route = (get & pathPrefix("box" / "byAddress") & path(Segment) & paging) { (address, offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + addressEncoder.fromString(address) match { + case Success(addr) => ApiResponse(getBoxesByAddress(addr, offset, limit)) + case Failure(_) => BadRequest("Incorrect address format") + } } } - private def getBoxesByAddressUnspent(addr: ErgoAddress, lastN: Long): Future[Seq[IndexedErgoBox]] = + private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Long, limit: Long): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => addr.retrieveBoxes(history, lastN).utxos() + case Some(addr) => addr.retrieveUtxos(history, offset, limit) case None => Seq.empty[IndexedErgoBox] } } - private def getBoxesByAddressUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byAddress") & path(Segment) & lastN) { (address, limit) => - addressEncoder.fromString(address) match { - case Success(addr) => ApiResponse(getBoxesByAddressUnspent(addr, limit)) - case Failure(_) => BadRequest("Incorrect address format") + private def getBoxesByAddressUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byAddress") & path(Segment) & paging) { (address, offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + addressEncoder.fromString(address) match { + case Success(addr) => ApiResponse(getBoxesByAddressUnspent(addr, offset, limit)) + case Failure(_) => BadRequest("Incorrect address format") + } } } - private def getBoxRange(fromHeight: Long, toHeight: Long): Future[Seq[ModifierId]] = + private def getBoxRange(offset: Long, limit: Long): Future[Seq[ModifierId]] = getHistory.map { history => - (for(n <- fromHeight to toHeight) - yield history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(n))).get.m - ).toSeq + val base: Int = (history.fullBlockHeight - offset).toInt + for(n <- (base - limit) to base) yield history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(n))).get.m } - private def getBoxRangeR: Route = (pathPrefix("box" / "range") & paging) { (fromIndex, toIndex) => - if (toIndex < fromIndex) { - BadRequest("toIndex < fromIndex") - } else if (fromIndex - toIndex > MaxBoxes) { - BadRequest(s"No more than $MaxBoxes boxes can be requested") - } else { - ApiResponse(getBoxRange(fromIndex, toIndex)) + private def getBoxRangeR: Route = (pathPrefix("box" / "range") & paging) { (offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + ApiResponse(getBoxRange(offset, limit)) } } - private def getBoxesByErgoTree(tree: ErgoTree, lastN: Long): Future[Seq[IndexedErgoBox]] = + private def getBoxesByErgoTree(tree: ErgoTree, offset: Long, limit: Long): Future[Seq[IndexedErgoBox]] = getHistory.map { history => history.typedModifierById[IndexedErgoTree](bytesToId(IndexedErgoTreeSerializer.ergoTreeHash(tree))) match { - case Some(iEt) => iEt.retrieveBody(history, lastN) + case Some(iEt) => iEt.retrieveBoxes(history, offset, limit) case None => Seq.empty[IndexedErgoBox] } } - private def getBoxesByErgoTreeR: Route = (get & pathPrefix("box" / "byErgoTree") & path(Segment) & lastN) { (tree, limit) => + private def getBoxesByErgoTreeR: Route = (get & pathPrefix("box" / "byErgoTree") & path(Segment) & paging) { (tree, offset, limit) => try { - ApiResponse(getBoxesByErgoTree(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(Base16.decode(tree).get), limit)) + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + ApiResponse(getBoxesByErgoTree(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(Base16.decode(tree).get), offset, limit)) + } }catch { case e: Exception => BadRequest(s"${e.getMessage}") } } - private def getBoxesByErgoTreeUnspent(tree: ErgoTree, lastN: Long): Future[Seq[IndexedErgoBox]] = + private def getBoxesByErgoTreeUnspent(tree: ErgoTree, offset: Long, limit: Long): Future[Seq[IndexedErgoBox]] = getHistory.map { history => history.typedModifierById[IndexedErgoTree](bytesToId(IndexedErgoTreeSerializer.ergoTreeHash(tree))) match { - case Some(iEt) => iEt.retrieveBody(history, lastN).filter(!_.trackedBox.isSpent) + case Some(iEt) => iEt.retrieveUtxos(history, offset, limit) case None => Seq.empty[IndexedErgoBox] } } - private def getBoxesByErgoTreeUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byErgoTree") & path(Segment) & lastN) { (tree, limit) => + private def getBoxesByErgoTreeUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byErgoTree") & path(Segment) & paging) { (tree, offset, limit) => try { - ApiResponse(getBoxesByErgoTreeUnspent(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(Base16.decode(tree).get), limit)) + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + ApiResponse(getBoxesByErgoTreeUnspent(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(Base16.decode(tree).get), offset, limit)) + } }catch { case e: Exception => BadRequest(s"${e.getMessage}") } diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index c61a1ccde2..e31052b79e 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -365,9 +365,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * @return available peers to download headers from together with the state/origin of the peer */ private def getPeersForDownloadingHeaders(callingPeer: ConnectedPeer): Iterable[ConnectedPeer] = { - syncTracker.peersByStatus - .get(Older) - .getOrElse(Array(callingPeer)) + syncTracker.peersByStatus.getOrElse(Older, Array(callingPeer)) } /** @@ -385,6 +383,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, // if the peer is not ok (e.g. of some old version having problems) // choose first peer which is okay // so usually returns randomized peer, with fallback to deterministic one + if(peers.isEmpty) return None // if no peers are available after sudden network change (eg. VPN) val randomPeer = peers(Random.nextInt(peers.size)) if (filterOutFn(randomPeer)) { Some(randomPeer) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala index c23df29c40..00a669bb0d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala @@ -102,19 +102,19 @@ trait ErgoHistory modifier match { case fb: ErgoFullBlock => val nonMarkedIds = (fb.header.id +: fb.header.sectionIds.map(_._2)) - .filter(id => historyStorage.getIndex(validityKey(id)).isEmpty) + .filter(id => historyStorage.getIndex(validityKey(id)).isEmpty).toArray if (nonMarkedIds.nonEmpty) { historyStorage.insert( nonMarkedIds.map(id => validityKey(id) -> Array(1.toByte)), - Nil).map(_ => this) + Array.empty[BlockSection]).map(_ => this) } else { Success(this) } case _ => historyStorage.insert( Array(validityKey(modifier.id) -> Array(1.toByte)), - Nil).map(_ => this) + Array.empty[BlockSection]).map(_ => this) } } @@ -131,17 +131,17 @@ trait ErgoHistory log.warn(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as invalid") correspondingHeader(modifier) match { case Some(invalidatedHeader) => - val invalidatedHeaders = continuationHeaderChains(invalidatedHeader, _ => true).flatten.distinct + val invalidatedHeaders = continuationHeaderChains(invalidatedHeader, _ => true).flatten.distinct.toArray val invalidatedIds = invalidatedHeaders.map(_.id).toSet val validityRow = invalidatedHeaders.flatMap(h => Seq(h.id, h.transactionsId, h.ADProofsId) .map(id => validityKey(id) -> Array(0.toByte))) - log.info(s"Going to invalidate ${invalidatedHeader.encodedId} and ${invalidatedHeaders.map(_.encodedId)}") + log.info(s"Going to invalidate ${invalidatedHeader.encodedId} and ${invalidatedHeaders.map(_.encodedId).mkString("Array(", ", ", ")")}") val bestHeaderIsInvalidated = bestHeaderIdOpt.exists(id => invalidatedIds.contains(id)) val bestFullIsInvalidated = bestFullBlockIdOpt.exists(id => invalidatedIds.contains(id)) (bestHeaderIsInvalidated, bestFullIsInvalidated) match { case (false, false) => // Modifiers from best header and best full chain are not involved, no rollback and links change required - historyStorage.insert(validityRow, Nil).map { _ => + historyStorage.insert(validityRow, Array.empty[BlockSection]).map { _ => this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) } case _ => @@ -151,8 +151,8 @@ trait ErgoHistory if (!bestFullIsInvalidated) { //Only headers chain involved historyStorage.insert( - newBestHeaderOpt.map(h => BestHeaderKey -> idToBytes(h.id)).toSeq, - Seq.empty + newBestHeaderOpt.map(h => BestHeaderKey -> idToBytes(h.id)).toArray, + Array.empty[BlockSection] ).map { _ => this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) } @@ -180,7 +180,7 @@ trait ErgoHistory val changedLinks = validHeadersChain.lastOption.map(b => BestFullBlockKey -> idToBytes(b.id)) ++ newBestHeaderOpt.map(h => BestHeaderKey -> idToBytes(h.id)).toSeq val toInsert = validityRow ++ changedLinks ++ chainStatusRow - historyStorage.insert(toInsert, Seq.empty).map { _ => + historyStorage.insert(toInsert, Array.empty[BlockSection]).map { _ => val toRemove = if (genesisInvalidated) invalidatedChain else invalidatedChain.tail this -> ProgressInfo(Some(branchPointHeader.id), toRemove, validChain, Seq.empty) } @@ -189,7 +189,7 @@ trait ErgoHistory case None => //No headers become invalid. Just mark this modifier as invalid log.warn(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is missing corresponding header") - historyStorage.insert(Array(validityKey(modifier.id) -> Array(0.toByte)), Nil).map { _ => + historyStorage.insert(Array(validityKey(modifier.id) -> Array(0.toByte)), Array.empty[BlockSection]).map { _ => this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) } } @@ -269,7 +269,7 @@ object ErgoHistory extends ScorexLogging { afterHeaders.map { hId => history.forgetHeader(hId) } - history.historyStorage.remove(Array(history.heightIdsKey(bestHeaderHeight + 1)), Nil) + history.historyStorage.remove(Array(history.heightIdsKey(bestHeaderHeight + 1)), Array.empty[ModifierId]) true } else { false diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 95a1684ce6..844d7e8181 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -3,7 +3,7 @@ package org.ergoplatform.nodeView.history.extra import akka.actor.{Actor, ActorRef, ActorSystem, Props} import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, Input} -import org.ergoplatform.modifiers.history.{BlockTransactions, HistoryModifierSerializer} +import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} @@ -20,19 +20,19 @@ import spire.syntax.all.cfor class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) extends Actor with ScorexLogging { - private val IndexedHeightKey: ByteArrayWrapper = ByteArrayWrapper.apply(Algos.hash("indexed height")) + private val IndexedHeightKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("indexed height")).data private var indexedHeight: Int = 0 private val indexedHeightBuffer : ByteBuffer = ByteBuffer.allocate(4) - private val GlobalTxIndexKey: ByteArrayWrapper = ByteArrayWrapper.apply(Algos.hash("txns height")) + private val GlobalTxIndexKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("txns height")).data private var globalTxIndex: Long = 0L private val globalTxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) - private val GlobalBoxIndexKey: ByteArrayWrapper = ByteArrayWrapper.apply(Algos.hash("boxes height")) + private val GlobalBoxIndexKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("boxes height")).data private var globalBoxIndex: Long = 0L private val globalBoxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) - private val saveLimit: Int = cacheSettings.history.extraCacheSize * 25 + private val saveLimit: Int = cacheSettings.history.extraCacheSize * 10 private var done: Boolean = false @@ -46,7 +46,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private val modifiers: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] private def findModifierOpt(id: Array[Byte]): Option[Int] = { - cfor(modifiers.size - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first + cfor(modifiers.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first if(java.util.Arrays.equals(modifiers(i).serializedId, id)) return Some(i) } None @@ -56,17 +56,25 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) val start: Long = System.nanoTime() - cfor(0)(_ < modifiers.length, _ + 1) { i => - historyStorage.insert(modifiers(i).serializedId, HistoryModifierSerializer.toBytes(modifiers(i))) + // perform segmentation on big modifiers + val len: Int = modifiers.length + cfor(0)(_ < len, _ + 1) { i => + modifiers(i) match { + case m: IndexedErgoAddress if m.txs.length > IndexedErgoAddress.segmentTreshold && m.boxes.length > IndexedErgoAddress.segmentTreshold => + modifiers += modifiers(i).asInstanceOf[IndexedErgoAddress].splitToSegment() + case m: IndexedErgoTree if m.boxes.length > IndexedErgoTree.segmentTreshold => + modifiers += modifiers(i).asInstanceOf[IndexedErgoTree].splitToSegment() + case _ => + } } + // insert modifiers and progress info to db indexedHeightBuffer.clear() globalTxIndexBuffer.clear() globalBoxIndexBuffer.clear() - - historyStorage.insert(Seq((IndexedHeightKey , indexedHeightBuffer .putInt (indexedHeight ).array), - (GlobalTxIndexKey , globalTxIndexBuffer .putLong(globalTxIndex ).array), - (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), Seq.empty) + historyStorage.insertExtra(Array((IndexedHeightKey , indexedHeightBuffer .putInt (indexedHeight ).array), + (GlobalTxIndexKey , globalTxIndexBuffer .putLong(globalTxIndex ).array), + (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), modifiers.toArray) val end: Long = System.nanoTime() @@ -78,18 +86,15 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private def index(bt: BlockTransactions, height: Int): Unit = { - //process transactions - cfor(0)(_ < bt.txIds.size, _ + 1) { i => - modifiers += IndexedErgoTransaction(bytesToId(bt.txIds(i)), indexedHeight, globalTxIndex) - modifiers += NumericTxIndex(globalTxIndex, bytesToId(bt.txIds(i))) - globalTxIndex += 1 - } - cfor(0)(_ < bt.txs.size, _ + 1) { n => val tx: ErgoTransaction = bt.txs(n) - //process tx inputs + //process transaction + modifiers += IndexedErgoTransaction(tx.id, indexedHeight, globalTxIndex) + modifiers += NumericTxIndex(globalTxIndex, tx.id) + + //process transaction inputs if(indexedHeight != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => val input: Input = tx.inputs(i) @@ -104,51 +109,57 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } } - //process tx outputs + //process transaction outputs cfor(0)(_ < tx.outputs.size, _ + 1) { i => val box: ErgoBox = tx.outputs(i) modifiers += new IndexedErgoBox(Some(indexedHeight), None, None, box, globalBoxIndex, Some(chainHeight - indexedHeight)) // box by id modifiers += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number - globalBoxIndex += 1 // box by ergotree val treeHash: Array[Byte] = IndexedErgoTreeSerializer.ergoTreeHash(box.ergoTree) findModifierOpt(treeHash) match { - case Some(x) => modifiers(x).asInstanceOf[IndexedErgoTree].addBox(bytesToId(box.id)) // ergotree found in last saveLimit modifiers, update + case Some(x) => modifiers(x).asInstanceOf[IndexedErgoTree].addBox(globalBoxIndex) // ergotree found in last saveLimit modifiers, update case None => // ergotree not found in last saveLimit blocks history.typedModifierById[IndexedErgoTree](bytesToId(treeHash)) match { - case Some(x) => modifiers += x.addBox(bytesToId(box.id)) // ergotree found in DB, update - case None => modifiers += IndexedErgoTree(bytesToId(treeHash), ListBuffer(bytesToId(box.id))) // ergotree not found at all, record + case Some(x) => modifiers += x.addBox(globalBoxIndex) // ergotree found in DB, update + case None => modifiers += IndexedErgoTree(bytesToId(treeHash), ListBuffer(globalBoxIndex)) // ergotree not found at all, record } } // box by address val addrHash: Array[Byte] = IndexedErgoAddressSerializer.hashAddress(IndexedErgoBoxSerializer.getAddress(box.ergoTree)) findModifierOpt(addrHash) match { - case Some(x) => modifiers(x).asInstanceOf[IndexedErgoAddress].addBox(box.id) // address found in last saveLimit modifiers, update + case Some(x) => modifiers(x).asInstanceOf[IndexedErgoAddress].addTx(globalTxIndex).addBox(globalBoxIndex) // address found in last saveLimit modifiers, update case None => // address not found in last saveLimit blocks history.typedModifierById[IndexedErgoAddress](bytesToId(addrHash)) match { - case Some(x) => modifiers += x.addTx(tx.id).addBox(box.id)//address found in DB, update - case None => modifiers += IndexedErgoAddress(bytesToId(addrHash), ListBuffer(tx.id), ListBuffer(box.id)) //address not found at all, record + case Some(x) => modifiers += x.addTx(globalTxIndex).addBox(globalBoxIndex)//address found in DB, update + case None => modifiers += IndexedErgoAddress(bytesToId(addrHash), ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) //address not found at all, record } } + + // TODO token indexing + + globalBoxIndex += 1 + } + globalTxIndex += 1 + } log.info(s"Indexed block #$indexedHeight [transactions: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] - progress: $indexedHeight / $chainHeight (mods: ${modifiers.size})") if(done) indexedHeight = height // update height here after caught up with chain - if(modifiers.size >= saveLimit || done) saveProgress() // save index every saveLimit modifiers or every block after caught up + if(modifiers.length >= saveLimit || done) saveProgress() // save index every saveLimit modifiers or every block after caught up } private def run(): Unit = { - indexedHeight = ByteBuffer.wrap(historyStorage.getIndex(IndexedHeightKey ).getOrElse(Array.fill[Byte](4){0})).getInt - globalTxIndex = ByteBuffer.wrap(historyStorage.getIndex(GlobalTxIndexKey ).getOrElse(Array.fill[Byte](8){0})).getLong - globalBoxIndex = ByteBuffer.wrap(historyStorage.getIndex(GlobalBoxIndexKey).getOrElse(Array.fill[Byte](8){0})).getLong + indexedHeight = ByteBuffer.wrap(historyStorage.get(bytesToId(IndexedHeightKey)) .getOrElse(Array.fill[Byte](4){0})).getInt + globalTxIndex = ByteBuffer.wrap(historyStorage.get(bytesToId(GlobalTxIndexKey)) .getOrElse(Array.fill[Byte](8){0})).getLong + globalBoxIndex = ByteBuffer.wrap(historyStorage.get(bytesToId(GlobalBoxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong log.info(s"Started extra indexer at height $indexedHeight") diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 05f7789e26..354b753965 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -2,22 +2,22 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoAddress import org.ergoplatform.ErgoAddressEncoder.{ChecksumLength, hash256} -import org.ergoplatform.ErgoBox.BoxId import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getSegmentsForRange, segmentTreshold, slice} +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.segmentId import org.ergoplatform.settings.Algos import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer -import scorex.crypto.authds.ADKey -import scorex.util.{ByteArrayOps, ModifierId, bytesToId, idToBytes} +import scorex.util.{ModifierId, bytesToId, idToBytes} import scorex.util.serialization.{Reader, Writer} import scala.collection.mutable.ListBuffer import spire.syntax.all.cfor case class IndexedErgoAddress(addressHash: ModifierId, - txIds: ListBuffer[ModifierId], - boxIds: ListBuffer[BoxId]) extends BlockSection { + txs: ListBuffer[Long], + boxes: ListBuffer[Long]) extends BlockSection { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = idToBytes(addressHash) @@ -26,55 +26,65 @@ case class IndexedErgoAddress(addressHash: ModifierId, override type M = IndexedErgoAddress override def serializer: ScorexSerializer[IndexedErgoAddress] = IndexedErgoAddressSerializer - private var _transactions: ListBuffer[IndexedErgoTransaction] = ListBuffer.empty[IndexedErgoTransaction] - private var _boxes: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] - private var _utxos: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] - - def transactions(lastN: Long = 20L): ListBuffer[IndexedErgoTransaction] = - if(lastN > 0) - _transactions.slice(math.max((_transactions.size - lastN).toInt, 0), _transactions.size) - else - _transactions - - def boxes(): ListBuffer[IndexedErgoBox] = _boxes - - def utxos(): ListBuffer[IndexedErgoBox] = _utxos - - def retrieveBody(history: ErgoHistoryReader): IndexedErgoAddress = { - retrieveTxs(history, -1) - retrieveBoxes(history, -1) - this - } - - def retrieveTxs(history: ErgoHistoryReader, lastN: Long): IndexedErgoAddress = { - _transactions = ( - if(lastN > 0) - txIds.slice(math.max((txIds.size - lastN).toInt, 0), txIds.size) - else - txIds - ).map(history.typedModifierById[IndexedErgoTransaction](_).get.retrieveBody(history)) - this + private[extra] var segmentCount: Int = 0 + + def txCount(): Long = segmentTreshold * segmentCount + txs.length + def boxCount(): Long = segmentTreshold * segmentCount + boxes.length + + def retrieveTxs(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoTransaction] = + if(offset > txs.length) { + val range: (Int, Int) = getSegmentsForRange(offset, limit) + val data: ListBuffer[IndexedErgoTransaction] = + history.typedModifierById[IndexedErgoAddress](segmentId(addressHash, segmentCount - range._1)).get.txs.map(n => NumericTxIndex.getTxByNumber(history, n).get) + if(range._2 > 0) data ++= + history.typedModifierById[IndexedErgoAddress](segmentId(addressHash, segmentCount - range._2)).get.txs.map(n => NumericTxIndex.getTxByNumber(history, n).get) + data.toArray + } else { + slice(txs, offset, limit).map(n => NumericTxIndex.getTxByNumber(history, n).get).toArray + } + + def retrieveBoxes(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = + if(offset > boxes.length) { + val range: (Int, Int) = getSegmentsForRange(offset, limit) + val data: ListBuffer[IndexedErgoBox] = + history.typedModifierById[IndexedErgoAddress](segmentId(addressHash, segmentCount - range._1)).get.boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get) + if(range._2 > 0) data ++= + history.typedModifierById[IndexedErgoAddress](segmentId(addressHash, segmentCount - range._2)).get.boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get) + data.toArray + } else { + slice(boxes, offset, limit).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray + } + + def retrieveUtxos(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = { + val data: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] + data ++= boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) + var segment = segmentCount + while(data.length < limit && segment > 0) { + segment -= 1 + data ++=: history.typedModifierById[IndexedErgoAddress](segmentId(addressHash, segment)).get.boxes + .map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) + } + slice(data, offset, limit).toArray } - def retrieveBoxes(history: ErgoHistoryReader, lastN: Long): IndexedErgoAddress = { - _boxes = ( - if(lastN > 0) - boxIds.slice(math.max((boxIds.size - lastN).toInt, 0), boxIds.size) - else - boxIds - ).map(x => history.typedModifierById[IndexedErgoBox](bytesToId(x)).get) - _utxos = _boxes.filter(!_.trackedBox.isSpent) + def addTx(tx: Long): IndexedErgoAddress = { + cfor(txs.length - 1)(_ >= 0, _ - 1) { i => if(txs(i) == tx) return this} // check for duplicates + txs += tx this } - def addTx(id: ModifierId): IndexedErgoAddress = { - txIds += id + def addBox(box: Long): IndexedErgoAddress = { + boxes += box this } - def addBox(box: BoxId): IndexedErgoAddress = { - boxIds += box - this + def splitToSegment(): IndexedErgoAddress = { + require(segmentTreshold < txs.length && segmentTreshold < boxes.length, "address does not have enough transactions or boxes for segmentation") + val iEa: IndexedErgoAddress = new IndexedErgoAddress(segmentId(addressHash, segmentCount), txs.take(segmentTreshold), boxes.take(segmentTreshold)) + segmentCount += 1 + txs.remove(0, segmentTreshold) + boxes.remove(0, segmentTreshold) + iEa } } @@ -89,26 +99,44 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] def addressToModifierId(address: ErgoAddress): ModifierId = bytesToId(hashAddress(address)) + def segmentId(addressHash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(addressHash + " segment " + segmentNum)) + override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = { w.putBytes(idToBytes(iEa.addressHash)) - w.putUInt(iEa.txIds.length) - cfor(0)(_ < iEa.txIds.length, _ + 1) { i => w.putBytes(iEa.txIds(i).toBytes)} - w.putUInt(iEa.boxIds.length) - cfor(0)(_ < iEa.boxIds.length, _ + 1) { i => w.putBytes(iEa.boxIds(i))} + w.putUInt(iEa.txs.length) + cfor(0)(_ < iEa.txs.length, _ + 1) { i => w.putLong(iEa.txs(i))} + w.putUInt(iEa.boxes.length) + cfor(0)(_ < iEa.boxes.length, _ + 1) { i => w.putLong(iEa.boxes(i))} + w.putInt(iEa.segmentCount) } override def parse(r: Reader): IndexedErgoAddress = { val addressHash: ModifierId = bytesToId(r.getBytes(32)) val txnsLen: Long = r.getUInt() - val txns: ListBuffer[ModifierId] = ListBuffer.empty[ModifierId] - cfor(0)(_ < txnsLen, _ + 1) {_ => txns += r.getBytes(32).toModifierId} + val txns: ListBuffer[Long] = ListBuffer.empty[Long] + cfor(0)(_ < txnsLen, _ + 1) { _ => txns += r.getLong()} val boxesLen: Long = r.getUInt() - val boxes: ListBuffer[BoxId] = ListBuffer.empty[BoxId] - cfor(0)(_ < boxesLen, _ + 1) { _ => boxes += ADKey @@ r.getBytes(32)} - new IndexedErgoAddress(addressHash, txns, boxes) + val boxes: ListBuffer[Long] = ListBuffer.empty[Long] + cfor(0)(_ < boxesLen, _ + 1) { _ => boxes += r.getLong()} + val iEa: IndexedErgoAddress = new IndexedErgoAddress(addressHash, txns, boxes) + iEa.segmentCount = r.getInt() + iEa } } object IndexedErgoAddress { + val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 15.toByte + + val segmentTreshold: Int = 8192 + + def getSegmentsForRange(offset: Long, limit: Long): (Int, Int) = { + require(limit <= segmentTreshold, "limit exceeds segmentTreshold") + val x: Int = math.ceil(offset / segmentTreshold).toInt + (x, if(offset % segmentTreshold >= limit) -1 else x - 1) + } + + def slice[T](arr: Iterable[T], offset: Long, limit: Long): Iterable[T] = + arr.slice((arr.size - offset - limit).toInt, (arr.size - offset + 1).toInt) + } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala index 40555257cf..d19f0eda3d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala @@ -2,6 +2,8 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.extra.IndexedErgoTree.{getSegmentsForRange, segmentTreshold, slice} +import org.ergoplatform.nodeView.history.extra.IndexedErgoTreeSerializer.segmentId import org.ergoplatform.settings.Algos import scorex.core.{ModifierTypeId, idToBytes} import scorex.core.serialization.ScorexSerializer @@ -12,7 +14,7 @@ import sigmastate.Values.ErgoTree import scala.collection.mutable.ListBuffer import spire.syntax.all.cfor -case class IndexedErgoTree(treeHash: ModifierId, boxIds: ListBuffer[ModifierId]) extends BlockSection { +case class IndexedErgoTree(treeHash: ModifierId, boxes: ListBuffer[Long]) extends BlockSection { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = idToBytes(treeHash) override def parentId: ModifierId = null @@ -20,39 +22,81 @@ case class IndexedErgoTree(treeHash: ModifierId, boxIds: ListBuffer[ModifierId]) override type M = IndexedErgoTree override def serializer: ScorexSerializer[IndexedErgoTree] = IndexedErgoTreeSerializer - def retrieveBody(history: ErgoHistoryReader, lastN: Long): Seq[IndexedErgoBox] = - ( - if(lastN > 0) - boxIds.slice(math.max((boxIds.size - lastN).toInt, 0), boxIds.size) - else - boxIds - ).map(history.typedModifierById[IndexedErgoBox](_).get) + private[extra] var segmentCount: Int = 0 - def addBox(id: ModifierId): IndexedErgoTree = { - boxIds += id + def boxCount(): Long = segmentTreshold * segmentCount + boxes.length + + def retrieveBoxes(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = + if(offset > boxes.length) { + val range: (Int, Int) = getSegmentsForRange(offset, limit) + val data: ListBuffer[IndexedErgoBox] = + history.typedModifierById[IndexedErgoAddress](segmentId(treeHash, segmentCount - range._1)).get.boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get) + if(range._2 > 0) data ++= + history.typedModifierById[IndexedErgoAddress](segmentId(treeHash, segmentCount - range._2)).get.boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get) + data.toArray + } else { + slice(boxes, offset, limit).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray + } + + def retrieveUtxos(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = { + val data: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] + data ++= boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) + var segment = segmentCount + while(data.length < limit && segment > 0) { + segment -= 1 + data ++=: history.typedModifierById[IndexedErgoAddress](segmentId(treeHash, segment)).get.boxes + .map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) + } + slice(data, offset, limit).toArray + } + + + def addBox(num: Long): IndexedErgoTree = { + boxes += num this } + + def splitToSegment(): IndexedErgoTree = { + require(segmentTreshold < boxes.length, "ergotree does not have enough boxes for segmentation") + val iEt: IndexedErgoTree = IndexedErgoTree(segmentId(treeHash, segmentCount), boxes.take(segmentTreshold)) + segmentCount += 1 + boxes.remove(0, segmentTreshold) + iEt + } } object IndexedErgoTreeSerializer extends ScorexSerializer[IndexedErgoTree] { def ergoTreeHash(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) + def segmentId(treeHash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(treeHash + " segment " + segmentNum)) + override def serialize(iEt: IndexedErgoTree, w: Writer): Unit = { w.putBytes(iEt.serializedId) - w.putUInt(iEt.boxIds.length) - cfor(0)(_ < iEt.boxIds.length, _ + 1) { i => w.putBytes(idToBytes(iEt.boxIds(i)))} + w.putLong(iEt.boxes.length) + cfor(0)(_ < iEt.boxes.length, _ + 1) { i => w.putLong(iEt.boxes(i))} + w.putInt(iEt.segmentCount) } override def parse(r: Reader): IndexedErgoTree = { val treeHash: ModifierId = bytesToId(r.getBytes(32)) - val boxIdsLen: Long = r.getUInt() - val boxIds: ListBuffer[ModifierId] = ListBuffer.empty[ModifierId] - cfor(0)(_ < boxIdsLen, _ + 1) { _ => boxIds += bytesToId(r.getBytes(32))} - IndexedErgoTree(treeHash, boxIds) + val boxesLen: Long = r.getLong() + val boxes: ListBuffer[Long] = ListBuffer.empty[Long] + cfor(0)(_ < boxesLen, _ + 1) { _ => boxes += r.getLong()} + val iEt: IndexedErgoTree = IndexedErgoTree(treeHash, boxes) + iEt.segmentCount = r.getInt() + iEt } } object IndexedErgoTree { + val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 20.toByte + + val segmentTreshold: Int = IndexedErgoAddress.segmentTreshold + + def getSegmentsForRange(offset: Long, limit: Long): (Int, Int) = IndexedErgoAddress.getSegmentsForRange(offset, limit) + + def slice[T](arr: Iterable[T], offset: Long, limit: Long): Iterable[T] = IndexedErgoAddress.slice(arr, offset, limit) + } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala index 4dda4d988a..e4185289e3 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala @@ -1,6 +1,7 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.settings.Algos import scorex.core.{ModifierTypeId, idToBytes} import scorex.core.serialization.ScorexSerializer @@ -20,12 +21,12 @@ case class NumericTxIndex(n: Long, m: ModifierId) extends BlockSection { object NumericTxIndexSerializer extends ScorexSerializer[NumericTxIndex] { override def serialize(ni: NumericTxIndex, w: Writer): Unit = { - w.putUInt(ni.n) + w.putLong(ni.n) w.putBytes(idToBytes(ni.m)) } override def parse(r: Reader): NumericTxIndex = { - val n: Long = r.getUInt() + val n: Long = r.getLong() val m: ModifierId = bytesToId(r.getBytes(32)) NumericTxIndex(n, m) } @@ -35,6 +36,9 @@ object NumericTxIndex { val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 25.toByte def indexToBytes(n: Long): Array[Byte] = Algos.hash("txns height " + n) + + def getTxByNumber(history: ErgoHistoryReader, n: Long): Option[IndexedErgoTransaction] = + history.typedModifierById[IndexedErgoTransaction](history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(n))).get.m) } case class NumericBoxIndex(n: Long, m: ModifierId) extends BlockSection { @@ -50,12 +54,12 @@ case class NumericBoxIndex(n: Long, m: ModifierId) extends BlockSection { object NumericBoxIndexSerializer extends ScorexSerializer[NumericBoxIndex] { override def serialize(ni: NumericBoxIndex, w: Writer): Unit = { - w.putUInt(ni.n) + w.putLong(ni.n) w.putBytes(idToBytes(ni.m)) } override def parse(r: Reader): NumericBoxIndex = { - val n: Long = r.getUInt() + val n: Long = r.getLong() val m: ModifierId = bytesToId(r.getBytes(32)) NumericBoxIndex(n, m) } @@ -65,4 +69,7 @@ object NumericBoxIndex { val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 30.toByte def indexToBytes(n: Long): Array[Byte] = Algos.hash("boxes height " + n) + + def getBoxByNumber(history: ErgoHistoryReader, n: Long): Option[IndexedErgoBox] = + history.typedModifierById[IndexedErgoBox](history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(n))).get.m) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 6d41f00fb0..fce441aeb0 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -4,13 +4,14 @@ import com.google.common.cache.CacheBuilder import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.modifiers.history.HistoryModifierSerializer import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoBox, IndexedErgoTree} +import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoTree} import org.ergoplatform.settings.{Algos, CacheSettings, ErgoSettings} import scorex.core.utils.ScorexEncoding import scorex.db.{ByteArrayWrapper, LDBFactory, LDBKVStore} import scorex.util.{ModifierId, ScorexLogging, idToBytes} import scala.util.{Failure, Success, Try} +import spire.syntax.all.cfor /** * Storage for Ergo history @@ -20,7 +21,7 @@ import scala.util.{Failure, Success, Try} * @param objectsStore - key-value store, where key is id of ErgoPersistentModifier and value is it's bytes * @param config - cache configs */ -class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, config: CacheSettings) +class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStore: LDBKVStore, config: CacheSettings) extends ScorexLogging with AutoCloseable with ScorexEncoding { @@ -43,7 +44,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c private def cacheModifier(mod: BlockSection): Unit = mod.modifierTypeId match { case Header.modifierTypeId => headersCache.put(mod.id, mod) - case IndexedErgoBox.modifierTypeId | IndexedErgoAddress.modifierTypeId | IndexedErgoTree.modifierTypeId => extraCache.put(mod.id, mod) + case IndexedErgoAddress.modifierTypeId | IndexedErgoTree.modifierTypeId => extraCache.put(mod.id, mod) case _ => blockSectionsCache.put(mod.id, mod) } @@ -62,7 +63,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c def modifierById(id: ModifierId): Option[BlockSection] = lookupModifier(id) orElse - objectsStore.get(idToBytes(id)).flatMap { bytes => + (extraStore.get(idToBytes(id)) orElse objectsStore.get(idToBytes(id))).flatMap { bytes => HistoryModifierSerializer.parseBytesTry(bytes) match { case Success(pm) => log.trace(s"Cache miss for existing modifier $id") @@ -86,21 +87,28 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c def contains(id: ModifierId): Boolean = objectsStore.get(idToBytes(id)).isDefined - def insert(indexesToInsert: Seq[(ByteArrayWrapper, Array[Byte])], - objectsToInsert: Seq[BlockSection]): Try[Unit] = { + def insert(indexesToInsert: Array[(ByteArrayWrapper, Array[Byte])], + objectsToInsert: Array[BlockSection]): Try[Unit] = { objectsStore.insert( - objectsToInsert.map(m => idToBytes(m.id) -> HistoryModifierSerializer.toBytes(m)) + objectsToInsert.map(m => m.serializedId -> HistoryModifierSerializer.toBytes(m)) ).flatMap { _ => - objectsToInsert.foreach(o => cacheModifier(o)) + cfor(0)(_ < objectsToInsert.length, _ + 1) { i => cacheModifier(objectsToInsert(i))} if (indexesToInsert.nonEmpty) { indexStore.insert(indexesToInsert.map { case (k, v) => k.data -> v }).map { _ => - indexesToInsert.foreach(kv => indexCache.put(kv._1, kv._2)) + cfor(0)(_ < indexesToInsert.length, _ + 1) { i => indexCache.put(indexesToInsert(i)._1, indexesToInsert(i)._2)} () } } else Success(()) } } + def insertExtra(indexesToInsert: Array[(Array[Byte], Array[Byte])], + objectsToInsert: Array[BlockSection]): Unit = { + extraStore.insert(objectsToInsert.map(m => m.serializedId -> HistoryModifierSerializer.toBytes(m))) + cfor(0)(_ < objectsToInsert.length, _ + 1) { i => cacheModifier(objectsToInsert(i))} + cfor(0)(_ < indexesToInsert.length, _ + 1) { i => extraStore.insert(indexesToInsert(i)._1, indexesToInsert(i)._2)} + } + /** * Insert single object to database. This version allows for efficient insert * when identifier and bytes of object (i.e. modifier, a block section) are known. @@ -121,17 +129,13 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c * @param idsToRemove - identifiers of modifiers to remove * @return */ - def remove(indicesToRemove: Seq[ByteArrayWrapper], - idsToRemove: Seq[ModifierId]): Try[Unit] = { + def remove(indicesToRemove: Array[ByteArrayWrapper], + idsToRemove: Array[ModifierId]): Try[Unit] = { objectsStore.remove(idsToRemove.map(idToBytes)).map { _ => - idsToRemove.foreach { id => - removeModifier(id) - } + cfor(0)(_ < idsToRemove.length, _ + 1) { i => removeModifier(idsToRemove(i))} indexStore.remove(indicesToRemove.map(_.data)).map { _ => - indicesToRemove.foreach { id => - indexCache.invalidate(id) - } + cfor(0)(_ < indicesToRemove.length, _ + 1) { i => indexCache.invalidate(indicesToRemove(i))} () } } @@ -149,6 +153,7 @@ object HistoryStorage { def apply(ergoSettings: ErgoSettings): HistoryStorage = { val indexStore = LDBFactory.createKvDb(s"${ergoSettings.directory}/history/index") val objectsStore = LDBFactory.createKvDb(s"${ergoSettings.directory}/history/objects") - new HistoryStorage(indexStore, objectsStore, ergoSettings.cacheSettings) + val extraStore = LDBFactory.createKvDb(s"${ergoSettings.directory}/history/extra") + new HistoryStorage(indexStore, objectsStore, extraStore, ergoSettings.cacheSettings) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala index c17816d69b..b86cd81f3d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala @@ -11,7 +11,6 @@ import scorex.util.{ModifierId, bytesToId, idToBytes} import scala.annotation.tailrec import scala.collection.immutable.TreeMap -import scala.collection.mutable import scala.util.{Failure, Success, Try} /** @@ -135,7 +134,7 @@ trait FullBlockProcessor extends HeadersProcessor { } //Orphaned block or full chain is not initialized yet logStatus(Seq(), Seq(), params.fullBlock, None) - historyStorage.insert(Seq.empty, Seq(params.newModRow)).map { _ => + historyStorage.insert(Array.empty[(ByteArrayWrapper, Array[Byte])], Array(params.newModRow)).map { _ => ProgressInfo(None, Seq.empty, Seq.empty, Seq.empty) } } @@ -230,17 +229,17 @@ trait FullBlockProcessor extends HeadersProcessor { } private def pruneBlockDataAt(heights: Seq[Int]): Try[Unit] = { - val toRemove: Seq[ModifierId] = heights.flatMap(h => headerIdsAtHeight(h)) + val toRemove: Array[ModifierId] = heights.flatMap(h => headerIdsAtHeight(h)) .flatMap(id => typedModifierById[Header](id)) - .flatMap(_.sectionIds.map(_._2)) - historyStorage.remove(mutable.WrappedArray.empty, toRemove) + .flatMap(_.sectionIds.map(_._2)).toArray + historyStorage.remove(Array.empty, toRemove) } private def updateStorage(newModRow: BlockSection, bestFullHeaderId: ModifierId, additionalIndexes: Seq[(ByteArrayWrapper, Array[Byte])]): Try[Unit] = { - val indicesToInsert = Seq(BestFullBlockKey -> idToBytes(bestFullHeaderId)) ++ additionalIndexes - historyStorage.insert(indicesToInsert, Seq(newModRow)).flatMap { _ => + val indicesToInsert = Array(BestFullBlockKey -> idToBytes(bestFullHeaderId)) ++ additionalIndexes + historyStorage.insert(indicesToInsert, Array(newModRow)).flatMap { _ => if (headersHeight >= fullBlockHeight) Success(()) else diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala index 9e207d8a12..05b344be92 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala @@ -3,12 +3,13 @@ package org.ergoplatform.nodeView.history.storage.modifierprocessors import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.modifiers.{NonHeaderBlockSection, ErgoFullBlock, BlockSection} +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NonHeaderBlockSection} import org.ergoplatform.settings.ValidationRules._ import org.ergoplatform.settings.{Algos, ErgoValidationSettings} import scorex.core.consensus.ProgressInfo import scorex.core.utils.ScorexEncoding import scorex.core.validation.{ModifierValidator, _} +import scorex.db.ByteArrayWrapper import scorex.util.ModifierId import scala.reflect.ClassTag @@ -82,7 +83,7 @@ trait FullBlockSectionProcessor extends BlockSectionProcessor with FullBlockProc } private def justPutToHistory(m: NonHeaderBlockSection): Try[ProgressInfo[BlockSection]] = { - historyStorage.insert(Seq.empty, Seq(m)).map { _ => + historyStorage.insert(Array.empty[(ByteArrayWrapper, Array[Byte])], Array[BlockSection](m)).map { _ => ProgressInfo(None, Seq.empty, Seq.empty, Seq.empty) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala index a93719e68d..ed7a7c7171 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala @@ -89,7 +89,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score * @return ProgressInfo - info required for State to be consistent with History */ protected def process(h: Header): Try[ProgressInfo[BlockSection]] = synchronized { - val dataToInsert: (Seq[(ByteArrayWrapper, Array[Byte])], Seq[BlockSection]) = toInsert(h) + val dataToInsert: (Array[(ByteArrayWrapper, Array[Byte])], Array[BlockSection]) = toInsert(h) historyStorage.insert(dataToInsert._1, dataToInsert._2).flatMap { _ => bestHeaderIdOpt match { @@ -107,7 +107,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score /** * Data to add to and remove from the storage to process this modifier */ - private def toInsert(h: Header): (Seq[(ByteArrayWrapper, Array[Byte])], Seq[BlockSection]) = { + private def toInsert(h: Header): (Array[(ByteArrayWrapper, Array[Byte])], Array[BlockSection]) = { val requiredDifficulty: Difficulty = h.requiredDifficulty val score = scoreOf(h.parentId).getOrElse(BigInt(0)) + requiredDifficulty val bestRow: Seq[(ByteArrayWrapper, Array[Byte])] = @@ -121,7 +121,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score orphanedBlockHeaderIdsRow(h, score) } - (Seq(scoreRow, heightRow) ++ bestRow ++ headerIdsRow, Seq(h)) + (Array(scoreRow, heightRow) ++ bestRow ++ headerIdsRow, Array(h)) } /** diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala index abdfbe04b9..fe1cbb237b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala @@ -48,7 +48,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco /** * Remove pre-3.3.0 derivation paths */ - def removePaths(): Try[Unit] = store.remove(Seq(SecretPathsKey)) + def removePaths(): Try[Unit] = store.remove(Array(SecretPathsKey)) /** * Store wallet-related public key in the database @@ -59,7 +59,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco store.insert { publicKeys.map { publicKey => pubKeyPrefixKey(publicKey) -> ExtendedPublicKeySerializer.toBytes(publicKey) - } + }.toArray } } @@ -99,7 +99,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco * @param ctx - state context */ def updateStateContext(ctx: ErgoStateContext): Try[Unit] = store - .insert(Seq(StateContextKey -> ctx.bytes)) + .insert(Array(StateContextKey -> ctx.bytes)) /** * Read state context from the database @@ -116,7 +116,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco */ def updateChangeAddress(address: P2PKAddress): Try[Unit] = { val bytes = settings.chainSettings.addressEncoder.toString(address).getBytes(Constants.StringEncoding) - store.insert(Seq(ChangeAddressKey -> bytes)) + store.insert(Array(ChangeAddressKey -> bytes)) } /** @@ -139,7 +139,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco def addScan(scanReq: ScanRequest): Try[Scan] = { val id = ScanId @@ (lastUsedScanId + 1).toShort scanReq.toScan(id).flatMap { app => - store.insert(Seq( + store.insert(Array( scanPrefixKey(id) -> ScanSerializer.toBytes(app), lastUsedScanIdKey -> Shorts.toByteArray(id) )).map(_ => app) @@ -151,7 +151,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco * @param id scan identifier */ def removeScan(id: Short): Try[Unit] = - store.remove(Seq(scanPrefixKey(id))) + store.remove(Array(scanPrefixKey(id))) /** * Get scan by its identifier diff --git a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala index 1be0f66f3c..3150b85e36 100644 --- a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala +++ b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala @@ -45,7 +45,7 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { ) settings.cacheSettings shouldBe CacheSettings( HistoryCacheSettings( - 12, 250, 100, 1000 + 12, 500, 100, 1000 ), NetworkCacheSettings( invalidModifiersBloomFilterCapacity = 10000000, @@ -97,7 +97,7 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { ) settings.cacheSettings shouldBe CacheSettings( HistoryCacheSettings( - 12, 250, 100, 1000 + 12, 500, 100, 1000 ), NetworkCacheSettings( invalidModifiersBloomFilterCapacity = 10000000, @@ -142,7 +142,7 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { ) settings.cacheSettings shouldBe CacheSettings( HistoryCacheSettings( - 12, 250, 100, 1000 + 12, 500, 100, 1000 ), NetworkCacheSettings( invalidModifiersBloomFilterCapacity = 10000000, From 241b88605178e415dbd44312475d81559330d2b7 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Tue, 23 Aug 2022 18:22:42 +0200 Subject: [PATCH 08/90] fixed variable retrieval, optimized addTx --- .../nodeView/history/extra/IndexedErgoAddress.scala | 3 +-- .../ergoplatform/nodeView/history/storage/HistoryStorage.scala | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 354b753965..d5084e8ad1 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -68,8 +68,7 @@ case class IndexedErgoAddress(addressHash: ModifierId, } def addTx(tx: Long): IndexedErgoAddress = { - cfor(txs.length - 1)(_ >= 0, _ - 1) { i => if(txs(i) == tx) return this} // check for duplicates - txs += tx + if(!txs.takeRight(5).contains(tx)) txs += tx // check for duplicates this } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index fce441aeb0..3e092cbf81 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -83,7 +83,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e } } - def get(id: ModifierId): Option[Array[Byte]] = objectsStore.get(idToBytes(id)) + def get(id: ModifierId): Option[Array[Byte]] = objectsStore.get(idToBytes(id)).orElse(extraStore.get(idToBytes(id))) def contains(id: ModifierId): Boolean = objectsStore.get(idToBytes(id)).isDefined From 25de33d6bd9f1f09e19769c0b3661ceb45de1642 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Wed, 24 Aug 2022 03:29:30 +0200 Subject: [PATCH 09/90] split load between multiple buffers, stricter segmentation --- .../http/api/BlockchainApiRoute.scala | 2 +- .../nodeView/history/extra/ExtraIndexer.scala | 116 +++++++++++++----- .../history/extra/IndexedErgoAddress.scala | 97 ++++++++------- .../history/extra/IndexedErgoBox.scala | 5 +- .../extra/IndexedErgoTransaction.scala | 5 +- .../history/extra/IndexedErgoTree.scala | 29 ++--- .../nodeView/history/extra/IndexedToken.scala | 5 +- .../nodeView/history/extra/NumericIndex.scala | 7 +- 8 files changed, 158 insertions(+), 108 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index cf863daeff..2c21cd5524 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -26,7 +26,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting val paging: Directive[(Long, Long)] = parameters("offset".as[Long] ? 0L, "limit".as[Long] ? 10L) - private val MaxItems = IndexedErgoAddress.segmentTreshold + private val MaxItems = 16384 val addressEncoder: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 844d7e8181..a09ccc9bc4 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -8,10 +8,11 @@ import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.segmentTreshold import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.settings.{Algos, CacheSettings, ChainSettings} import scorex.db.ByteArrayWrapper -import scorex.util.{ScorexLogging, bytesToId} +import scorex.util.{ModifierId, ScorexLogging, bytesToId} import java.nio.ByteBuffer import scala.collection.mutable.{ArrayBuffer, ListBuffer} @@ -43,30 +44,55 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private def historyStorage: HistoryStorage = _history.historyStorage // fast access - private val modifiers: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] + private val general: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] + private val boxes: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] + private val trees: ArrayBuffer[IndexedErgoTree] = ArrayBuffer.empty[IndexedErgoTree] + private val addresses: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] + + private def findBoxOpt(id: Array[Byte]): Option[Int] = { + cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first + if(java.util.Arrays.equals(boxes(i).serializedId, id)) return Some(i) + } + None + } - private def findModifierOpt(id: Array[Byte]): Option[Int] = { - cfor(modifiers.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(java.util.Arrays.equals(modifiers(i).serializedId, id)) return Some(i) + private def findTreeOpt(id: Array[Byte]): Option[Int] = { + cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first + if(java.util.Arrays.equals(trees(i).serializedId, id)) return Some(i) } None } + private def findAddressOpt(id: Array[Byte]): Option[Int] = { + cfor(addresses.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first + if(java.util.Arrays.equals(addresses(i).serializedId, id)) return Some(i) + } + None + } + + private def modCount: Int = general.length + boxes.length + trees.length + addresses.length + private def saveProgress(): Unit = { val start: Long = System.nanoTime() // perform segmentation on big modifiers - val len: Int = modifiers.length - cfor(0)(_ < len, _ + 1) { i => - modifiers(i) match { - case m: IndexedErgoAddress if m.txs.length > IndexedErgoAddress.segmentTreshold && m.boxes.length > IndexedErgoAddress.segmentTreshold => - modifiers += modifiers(i).asInstanceOf[IndexedErgoAddress].splitToSegment() - case m: IndexedErgoTree if m.boxes.length > IndexedErgoTree.segmentTreshold => - modifiers += modifiers(i).asInstanceOf[IndexedErgoTree].splitToSegment() - case _ => - } + val treesLen: Int = trees.length + cfor(0)(_ < treesLen, _ + 1) { i => + if(trees(i).boxes.length > segmentTreshold) trees += trees(i).splitToSegment() } + val addressesLen: Int = addresses.length + cfor(0)(_ < addressesLen, _ + 1) { i => + if(addresses(i).txs.length > segmentTreshold || (addresses(i).boxes.length > segmentTreshold)) addresses ++= addresses(i).splitToSegment() + } + + // merge all modifiers to an Array, avoids reallocations durin concatenation (++) + val all: Array[BlockSection] = new Array[BlockSection](modCount) + val offset: Array[Int] = Array(0, general.length, general.length + boxes.length, general.length + boxes.length + trees.length) + cfor(0)(_ < general.length , _ + 1) { i => all(i + offset(0)) = general(i) } + cfor(0)(_ < boxes.length , _ + 1) { i => all(i + offset(1)) = boxes(i) } + cfor(0)(_ < trees.length , _ + 1) { i => all(i + offset(2)) = trees(i) } + cfor(0)(_ < addresses.length, _ + 1) { i => all(i + offset(3)) = addresses(i) } // insert modifiers and progress info to db indexedHeightBuffer.clear() @@ -74,13 +100,17 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) globalBoxIndexBuffer.clear() historyStorage.insertExtra(Array((IndexedHeightKey , indexedHeightBuffer .putInt (indexedHeight ).array), (GlobalTxIndexKey , globalTxIndexBuffer .putLong(globalTxIndex ).array), - (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), modifiers.toArray) + (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), all) val end: Long = System.nanoTime() - log.info(s"Wrote ${modifiers.size} extra indexes to database in ${(end - start) / 1000000000D}s") + log.info(s"Processed ${boxes.length} boxes, ${trees.length} ErgoTrees, ${addresses.length} addresses and inserted them to database in ${(end - start) / 1000000000D}s") - modifiers.clear() + // clear buffers for next batch + general.clear() + boxes.clear() + trees.clear() + addresses.clear() } @@ -91,18 +121,18 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) val tx: ErgoTransaction = bt.txs(n) //process transaction - modifiers += IndexedErgoTransaction(tx.id, indexedHeight, globalTxIndex) - modifiers += NumericTxIndex(globalTxIndex, tx.id) + general += IndexedErgoTransaction(tx.id, indexedHeight, globalTxIndex) + general += NumericTxIndex(globalTxIndex, tx.id) //process transaction inputs if(indexedHeight != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => val input: Input = tx.inputs(i) - findModifierOpt(input.boxId) match { - case Some(x) => modifiers(x).asInstanceOf[IndexedErgoBox].asSpent(tx.id, indexedHeight) // box found in last saveLimit modifiers, update + findBoxOpt(input.boxId) match { + case Some(x) => boxes(x).asSpent(tx.id, indexedHeight) // box found in last saveLimit modifiers, update case None => // box not found in last saveLimit blocks history.typedModifierById[IndexedErgoBox](bytesToId(input.boxId)) match { - case Some(x) => modifiers += x.asSpent(tx.id, indexedHeight) // box found in DB, update + case Some(x) => boxes += x.asSpent(tx.id, indexedHeight) // box found in DB, update case None => log.warn(s"Input for box ${bytesToId(input.boxId)} not found in database") // box not found at all (this shouldn't happen) } } @@ -112,28 +142,28 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) //process transaction outputs cfor(0)(_ < tx.outputs.size, _ + 1) { i => val box: ErgoBox = tx.outputs(i) - modifiers += new IndexedErgoBox(Some(indexedHeight), None, None, box, globalBoxIndex, Some(chainHeight - indexedHeight)) // box by id - modifiers += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number + boxes += new IndexedErgoBox(Some(indexedHeight), None, None, box, globalBoxIndex, Some(chainHeight - indexedHeight)) // box by id + general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number // box by ergotree val treeHash: Array[Byte] = IndexedErgoTreeSerializer.ergoTreeHash(box.ergoTree) - findModifierOpt(treeHash) match { - case Some(x) => modifiers(x).asInstanceOf[IndexedErgoTree].addBox(globalBoxIndex) // ergotree found in last saveLimit modifiers, update + findTreeOpt(treeHash) match { + case Some(x) => trees(x).addBox(globalBoxIndex) // ergotree found in last saveLimit modifiers, update case None => // ergotree not found in last saveLimit blocks history.typedModifierById[IndexedErgoTree](bytesToId(treeHash)) match { - case Some(x) => modifiers += x.addBox(globalBoxIndex) // ergotree found in DB, update - case None => modifiers += IndexedErgoTree(bytesToId(treeHash), ListBuffer(globalBoxIndex)) // ergotree not found at all, record + case Some(x) => trees += x.addBox(globalBoxIndex) // ergotree found in DB, update + case None => trees += IndexedErgoTree(bytesToId(treeHash), ListBuffer(globalBoxIndex)) // ergotree not found at all, record } } // box by address val addrHash: Array[Byte] = IndexedErgoAddressSerializer.hashAddress(IndexedErgoBoxSerializer.getAddress(box.ergoTree)) - findModifierOpt(addrHash) match { - case Some(x) => modifiers(x).asInstanceOf[IndexedErgoAddress].addTx(globalTxIndex).addBox(globalBoxIndex) // address found in last saveLimit modifiers, update + findAddressOpt(addrHash) match { + case Some(x) => addresses(x).addTx(globalTxIndex).addBox(globalBoxIndex) // address found in last saveLimit modifiers, update case None => // address not found in last saveLimit blocks history.typedModifierById[IndexedErgoAddress](bytesToId(addrHash)) match { - case Some(x) => modifiers += x.addTx(globalTxIndex).addBox(globalBoxIndex)//address found in DB, update - case None => modifiers += IndexedErgoAddress(bytesToId(addrHash), ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) //address not found at all, record + case Some(x) => addresses += x.addTx(globalTxIndex).addBox(globalBoxIndex)//address found in DB, update + case None => addresses += IndexedErgoAddress(bytesToId(addrHash), ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) //address not found at all, record } } @@ -147,11 +177,11 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } - log.info(s"Indexed block #$indexedHeight [transactions: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] - progress: $indexedHeight / $chainHeight (mods: ${modifiers.size})") + log.info(s"Buffered block #$indexedHeight / $chainHeight [txs: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] (buffer: $modCount / $saveLimit)") if(done) indexedHeight = height // update height here after caught up with chain - if(modifiers.length >= saveLimit || done) saveProgress() // save index every saveLimit modifiers or every block after caught up + if(modCount >= saveLimit || done) saveProgress() // save index every saveLimit modifiers or every block after caught up } @@ -203,6 +233,24 @@ object ExtraIndexerRef { def setAddressEncoder(encoder: ErgoAddressEncoder): Unit = if(_ae == null) _ae = encoder + private val hexIndex: Array[Byte] = { + val index = Array.fill[Byte](128)(0xff.toByte) + "0123456789abcdef".toCharArray.zipWithIndex.foreach { case (c, i) => + index(c) = i.toByte + } + "abcdef".toCharArray.foreach{ c => + index(c.toUpper) = index(c) + } + index + } + + // faster id to bytes - no safety checks + def fastIdToBytes(id: ModifierId): Array[Byte] = { + val x: Array[Byte] = Array.ofDim[Byte](id.length / 2) + cfor(0)(_ < x.length, _ + 2) {i => x(i / 2) = ((hexIndex(id(i)) << 4) | hexIndex(id(i + 1))).toByte} + x + } + def apply(chainSettings: ChainSettings, cacheSettings: CacheSettings)(implicit system: ActorSystem): ActorRef = { val actor = system.actorOf(Props.create(classOf[ExtraIndex], chainSettings, cacheSettings)) _ae = chainSettings.addressEncoder diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index d5084e8ad1..6016d47c5c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -4,12 +4,13 @@ import org.ergoplatform.ErgoAddress import org.ergoplatform.ErgoAddressEncoder.{ChecksumLength, hash256} import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getSegmentsForRange, segmentTreshold, slice} -import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.segmentId +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, txSegmentId} import org.ergoplatform.settings.Algos import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer -import scorex.util.{ModifierId, bytesToId, idToBytes} +import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} import scala.collection.mutable.ListBuffer @@ -20,48 +21,45 @@ case class IndexedErgoAddress(addressHash: ModifierId, boxes: ListBuffer[Long]) extends BlockSection { override val sizeOpt: Option[Int] = None - override def serializedId: Array[Byte] = idToBytes(addressHash) + override def serializedId: Array[Byte] = fastIdToBytes(addressHash) override def parentId: ModifierId = null override val modifierTypeId: ModifierTypeId = IndexedErgoAddress.modifierTypeId override type M = IndexedErgoAddress override def serializer: ScorexSerializer[IndexedErgoAddress] = IndexedErgoAddressSerializer - private[extra] var segmentCount: Int = 0 - - def txCount(): Long = segmentTreshold * segmentCount + txs.length - def boxCount(): Long = segmentTreshold * segmentCount + boxes.length - - def retrieveTxs(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoTransaction] = - if(offset > txs.length) { - val range: (Int, Int) = getSegmentsForRange(offset, limit) - val data: ListBuffer[IndexedErgoTransaction] = - history.typedModifierById[IndexedErgoAddress](segmentId(addressHash, segmentCount - range._1)).get.txs.map(n => NumericTxIndex.getTxByNumber(history, n).get) - if(range._2 > 0) data ++= - history.typedModifierById[IndexedErgoAddress](segmentId(addressHash, segmentCount - range._2)).get.txs.map(n => NumericTxIndex.getTxByNumber(history, n).get) - data.toArray - } else { - slice(txs, offset, limit).map(n => NumericTxIndex.getTxByNumber(history, n).get).toArray + private[extra] var boxSegmentCount: Int = 0 + private[extra] var txSegmentCount: Int = 0 + + def txCount(): Long = segmentTreshold * txSegmentCount + txs.length + def boxCount(): Long = segmentTreshold * boxSegmentCount + boxes.length + + def retrieveTxs(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoTransaction] = { + if (offset > txs.length) { + val range: Array[Int] = getSegmentsForRange(offset, limit) + cfor(0)(_ < range.length, _ + 1) { i => + txs ++=: history.typedModifierById[IndexedErgoAddress](txSegmentId(addressHash, txSegmentCount - range(i))).get.txs + } } + slice(txs, offset, limit).map(n => NumericTxIndex.getTxByNumber(history, n).get).toArray + } - def retrieveBoxes(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = + def retrieveBoxes(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = { if(offset > boxes.length) { - val range: (Int, Int) = getSegmentsForRange(offset, limit) - val data: ListBuffer[IndexedErgoBox] = - history.typedModifierById[IndexedErgoAddress](segmentId(addressHash, segmentCount - range._1)).get.boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get) - if(range._2 > 0) data ++= - history.typedModifierById[IndexedErgoAddress](segmentId(addressHash, segmentCount - range._2)).get.boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get) - data.toArray - } else { - slice(boxes, offset, limit).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray + val range: Array[Int] = getSegmentsForRange(offset, limit) + cfor(0)(_ < range.length, _ + 1) { i => + boxes ++=: history.typedModifierById[IndexedErgoAddress](boxSegmentId(addressHash, boxSegmentCount - range(i))).get.boxes + } } + slice(boxes, offset, limit).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray + } def retrieveUtxos(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = { val data: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] data ++= boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) - var segment = segmentCount + var segment = boxSegmentCount while(data.length < limit && segment > 0) { segment -= 1 - data ++=: history.typedModifierById[IndexedErgoAddress](segmentId(addressHash, segment)).get.boxes + data ++=: history.typedModifierById[IndexedErgoAddress](boxSegmentId(addressHash, segment)).get.boxes .map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) } slice(data, offset, limit).toArray @@ -77,13 +75,20 @@ case class IndexedErgoAddress(addressHash: ModifierId, this } - def splitToSegment(): IndexedErgoAddress = { - require(segmentTreshold < txs.length && segmentTreshold < boxes.length, "address does not have enough transactions or boxes for segmentation") - val iEa: IndexedErgoAddress = new IndexedErgoAddress(segmentId(addressHash, segmentCount), txs.take(segmentTreshold), boxes.take(segmentTreshold)) - segmentCount += 1 - txs.remove(0, segmentTreshold) - boxes.remove(0, segmentTreshold) - iEa + def splitToSegment(): Array[IndexedErgoAddress] = { + require(segmentTreshold < txs.length || segmentTreshold < boxes.length, "address does not have enough transactions or boxes for segmentation") + val data: ListBuffer[IndexedErgoAddress] = ListBuffer.empty[IndexedErgoAddress] + if(segmentTreshold < txs.length) { + data += new IndexedErgoAddress(txSegmentId(addressHash, txSegmentCount), txs.take(segmentTreshold), ListBuffer.empty[Long]) + txSegmentCount += 1 + txs.remove(0, segmentTreshold) + } + if(segmentTreshold < boxes.length) { + data += new IndexedErgoAddress(boxSegmentId(addressHash, segmentTreshold), ListBuffer.empty[Long], boxes.take(segmentTreshold)) + boxSegmentCount += 1 + boxes.remove(0, segmentTreshold) + } + data.toArray } } @@ -98,15 +103,17 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] def addressToModifierId(address: ErgoAddress): ModifierId = bytesToId(hashAddress(address)) - def segmentId(addressHash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(addressHash + " segment " + segmentNum)) + def boxSegmentId(addressHash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(addressHash + " box segment " + segmentNum)) + def txSegmentId(addressHash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(addressHash + " tx segment " + segmentNum)) override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = { - w.putBytes(idToBytes(iEa.addressHash)) + w.putBytes(fastIdToBytes(iEa.addressHash)) w.putUInt(iEa.txs.length) cfor(0)(_ < iEa.txs.length, _ + 1) { i => w.putLong(iEa.txs(i))} w.putUInt(iEa.boxes.length) cfor(0)(_ < iEa.boxes.length, _ + 1) { i => w.putLong(iEa.boxes(i))} - w.putInt(iEa.segmentCount) + w.putInt(iEa.boxSegmentCount) + w.putInt(iEa.txSegmentCount) } override def parse(r: Reader): IndexedErgoAddress = { @@ -118,7 +125,8 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] val boxes: ListBuffer[Long] = ListBuffer.empty[Long] cfor(0)(_ < boxesLen, _ + 1) { _ => boxes += r.getLong()} val iEa: IndexedErgoAddress = new IndexedErgoAddress(addressHash, txns, boxes) - iEa.segmentCount = r.getInt() + iEa.boxSegmentCount = r.getInt() + iEa.txSegmentCount = r.getInt() iEa } } @@ -127,13 +135,10 @@ object IndexedErgoAddress { val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 15.toByte - val segmentTreshold: Int = 8192 + val segmentTreshold: Int = 512 - def getSegmentsForRange(offset: Long, limit: Long): (Int, Int) = { - require(limit <= segmentTreshold, "limit exceeds segmentTreshold") - val x: Int = math.ceil(offset / segmentTreshold).toInt - (x, if(offset % segmentTreshold >= limit) -1 else x - 1) - } + def getSegmentsForRange(offset: Long, limit: Long): Array[Int] = + (math.ceil((offset + limit) * 1F / segmentTreshold).toInt to math.ceil((offset + 1F) / segmentTreshold).toInt by -1).toArray.reverse def slice[T](arr: Iterable[T], offset: Long, limit: Long): Iterable[T] = arr.slice((arr.size - offset - limit).toInt, (arr.size - offset + 1).toInt) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala index 3065ee3ca1..ad0c76d86a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -2,12 +2,13 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.{ErgoAddress, ErgoBox, Pay2SAddress} import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import org.ergoplatform.nodeView.wallet.WalletBox import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.boxes.{ErgoBoxSerializer, TrackedBox} import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer -import scorex.util.{ModifierId, bytesToId, idToBytes} +import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.ErgoTree @@ -51,7 +52,7 @@ object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { override def serialize(iEb: IndexedErgoBox, w: Writer): Unit = { w.putOption[Int](iEb.inclusionHeightOpt)(_.putInt(_)) - w.putOption[ModifierId](iEb.spendingTxIdOpt)((ww, id) => ww.putBytes(idToBytes(id))) + w.putOption[ModifierId](iEb.spendingTxIdOpt)((ww, id) => ww.putBytes(fastIdToBytes(id))) w.putOption[Int](iEb.spendingHeightOpt)(_.putInt(_)) ErgoBoxSerializer.serialize(iEb.box, w) w.putLong(iEb.globalIndex) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index 2ea2e003e9..768c2118ba 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -5,8 +5,9 @@ import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.DataInput +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import scorex.core.serialization.ScorexSerializer -import scorex.core.{ModifierTypeId, idToBytes} +import scorex.core.ModifierTypeId import scorex.util.serialization.{Reader, Writer} import scorex.util.{ModifierId, bytesToId} @@ -15,7 +16,7 @@ case class IndexedErgoTransaction(txid: ModifierId, globalIndex: Long) extends BlockSection { override val modifierTypeId: ModifierTypeId = IndexedErgoTransaction.modifierTypeId - override def serializedId: Array[Byte] = idToBytes(txid) + override def serializedId: Array[Byte] = fastIdToBytes(txid) override val sizeOpt: Option[Int] = None override def parentId: ModifierId = null override type M = IndexedErgoTransaction diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala index d19f0eda3d..2932f26002 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala @@ -2,10 +2,11 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader -import org.ergoplatform.nodeView.history.extra.IndexedErgoTree.{getSegmentsForRange, segmentTreshold, slice} +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getSegmentsForRange, segmentTreshold, slice} import org.ergoplatform.nodeView.history.extra.IndexedErgoTreeSerializer.segmentId import org.ergoplatform.settings.Algos -import scorex.core.{ModifierTypeId, idToBytes} +import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} @@ -16,7 +17,7 @@ import spire.syntax.all.cfor case class IndexedErgoTree(treeHash: ModifierId, boxes: ListBuffer[Long]) extends BlockSection { override val sizeOpt: Option[Int] = None - override def serializedId: Array[Byte] = idToBytes(treeHash) + override def serializedId: Array[Byte] = fastIdToBytes(treeHash) override def parentId: ModifierId = null override val modifierTypeId: ModifierTypeId = IndexedErgoTree.modifierTypeId override type M = IndexedErgoTree @@ -26,17 +27,15 @@ case class IndexedErgoTree(treeHash: ModifierId, boxes: ListBuffer[Long]) extend def boxCount(): Long = segmentTreshold * segmentCount + boxes.length - def retrieveBoxes(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = + def retrieveBoxes(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = { if(offset > boxes.length) { - val range: (Int, Int) = getSegmentsForRange(offset, limit) - val data: ListBuffer[IndexedErgoBox] = - history.typedModifierById[IndexedErgoAddress](segmentId(treeHash, segmentCount - range._1)).get.boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get) - if(range._2 > 0) data ++= - history.typedModifierById[IndexedErgoAddress](segmentId(treeHash, segmentCount - range._2)).get.boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get) - data.toArray - } else { - slice(boxes, offset, limit).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray + val range: Array[Int] = getSegmentsForRange(offset, limit) + cfor(0)(_ < range.length, _ + 1) { i => + boxes ++=: history.typedModifierById[IndexedErgoAddress](segmentId(treeHash, segmentCount - range(i))).get.boxes + } } + slice(boxes, offset, limit).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray + } def retrieveUtxos(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = { val data: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] @@ -93,10 +92,4 @@ object IndexedErgoTree { val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 20.toByte - val segmentTreshold: Int = IndexedErgoAddress.segmentTreshold - - def getSegmentsForRange(offset: Long, limit: Long): (Int, Int) = IndexedErgoAddress.getSegmentsForRange(offset, limit) - - def slice[T](arr: Iterable[T], offset: Long, limit: Long): Iterable[T] = IndexedErgoAddress.slice(arr, offset, limit) - } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index e3db27a962..7761f94b53 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -3,7 +3,8 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.{R4, R5, R6} import org.ergoplatform.modifiers.BlockSection -import scorex.core.{ModifierTypeId, idToBytes} +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes +import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} @@ -20,7 +21,7 @@ case class IndexedToken(tokenId: ModifierId, override type M = IndexedToken override def serializer: ScorexSerializer[IndexedToken] = IndexedTokenSerializer override val sizeOpt: Option[Int] = None - override def serializedId: Array[Byte] = idToBytes(tokenId) + override def serializedId: Array[Byte] = fastIdToBytes(tokenId) } object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala index e4185289e3..5b7a5bae37 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala @@ -2,8 +2,9 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import org.ergoplatform.settings.Algos -import scorex.core.{ModifierTypeId, idToBytes} +import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} @@ -22,7 +23,7 @@ object NumericTxIndexSerializer extends ScorexSerializer[NumericTxIndex] { override def serialize(ni: NumericTxIndex, w: Writer): Unit = { w.putLong(ni.n) - w.putBytes(idToBytes(ni.m)) + w.putBytes(fastIdToBytes(ni.m)) } override def parse(r: Reader): NumericTxIndex = { @@ -55,7 +56,7 @@ object NumericBoxIndexSerializer extends ScorexSerializer[NumericBoxIndex] { override def serialize(ni: NumericBoxIndex, w: Writer): Unit = { w.putLong(ni.n) - w.putBytes(idToBytes(ni.m)) + w.putBytes(fastIdToBytes(ni.m)) } override def parse(r: Reader): NumericBoxIndex = { From 1809afc81f9e6c2366d84bfb0706d9b8b5e5c191 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Mon, 29 Aug 2022 23:11:00 +0200 Subject: [PATCH 10/90] [UNTESTED] added token tracking, eliminated separate address based indexing --- .../org/ergoplatform/http/api/ApiCodecs.scala | 13 ++- .../http/api/BlockchainApiRoute.scala | 27 ++++-- .../history/HistoryModifierSerializer.scala | 7 +- .../nodeView/history/extra/ExtraIndexer.scala | 85 ++++++++--------- .../history/extra/IndexedErgoAddress.scala | 27 +++--- .../history/extra/IndexedErgoBox.scala | 9 +- .../history/extra/IndexedErgoTree.scala | 95 ------------------- .../nodeView/history/extra/IndexedToken.scala | 24 +++-- .../history/storage/HistoryStorage.scala | 4 +- 9 files changed, 102 insertions(+), 189 deletions(-) delete mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index c6724e4a41..7e532eebb0 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -28,7 +28,7 @@ import sigmastate.interpreter._ import sigmastate.interpreter.CryptoConstants.EcPointType import io.circe.syntax._ import org.ergoplatform.http.api.requests.{CryptoResult, ExecuteRequest, HintExtractionRequest} -import org.ergoplatform.nodeView.history.extra.{ExtraIndexerRef, IndexedErgoBox, IndexedErgoTransaction} +import org.ergoplatform.nodeView.history.extra.{ExtraIndexerRef, IndexedErgoBox, IndexedErgoTransaction, IndexedToken} import org.ergoplatform.wallet.interface4j.SecretString import scorex.crypto.authds.{LeafData, Side} import scorex.crypto.authds.merkle.MerkleProof @@ -493,6 +493,17 @@ trait ApiCodecs extends JsonCodecs { "size" -> tx.txSize.asJson) } + implicit val IndexedTokenEncoder: Encoder[IndexedToken] = { token => + Json.obj( + "id" -> token.id.asJson, + "boxId" -> token.boxId.asJson, + "emissionAmount" -> token.amount.asJson, + "name" -> token.name.asJson, + "description" -> token.description.asJson, + "decimals" -> token.decimals.asJson + ) + } + } trait ApiEncoderOption diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 2c21cd5524..597b473238 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -42,16 +42,19 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting getBoxesByAddressUnspentR ~ getBoxRangeR ~ getBoxesByErgoTreeR ~ - getBoxesByErgoTreeUnspentR + getBoxesByErgoTreeUnspentR ~ + getTokenInfoByIdR } private def getHistory: Future[ErgoHistoryReader] = (readersHolder ? GetDataFromHistory[ErgoHistoryReader](r => r)).mapTo[ErgoHistoryReader] - private def getAddress(addr: ErgoAddress)(implicit history: ErgoHistoryReader): Option[IndexedErgoAddress] = { - history.typedModifierById[IndexedErgoAddress](IndexedErgoAddressSerializer.addressToModifierId(addr)) + private def getAddress(tree: ErgoTree)(implicit history: ErgoHistoryReader): Option[IndexedErgoAddress] = { + history.typedModifierById[IndexedErgoAddress](bytesToId(IndexedErgoAddressSerializer.hashErgoTree(tree))) } + private def getAddress(addr: ErgoAddress)(implicit history: ErgoHistoryReader): Option[IndexedErgoAddress] = getAddress(addr.script) + private def getTxById(id: ModifierId)(implicit history: ErgoHistoryReader): Option[IndexedErgoTransaction] = history.typedModifierById[IndexedErgoTransaction](id) match { case Some(tx) => Some(tx.retrieveBody(history)) @@ -185,8 +188,8 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxesByErgoTree(tree: ErgoTree, offset: Long, limit: Long): Future[Seq[IndexedErgoBox]] = getHistory.map { history => - history.typedModifierById[IndexedErgoTree](bytesToId(IndexedErgoTreeSerializer.ergoTreeHash(tree))) match { - case Some(iEt) => iEt.retrieveBoxes(history, offset, limit) + getAddress(tree)(history) match { + case Some(iEa) => iEa.retrieveBoxes(history, offset, limit) case None => Seq.empty[IndexedErgoBox] } } @@ -205,8 +208,8 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxesByErgoTreeUnspent(tree: ErgoTree, offset: Long, limit: Long): Future[Seq[IndexedErgoBox]] = getHistory.map { history => - history.typedModifierById[IndexedErgoTree](bytesToId(IndexedErgoTreeSerializer.ergoTreeHash(tree))) match { - case Some(iEt) => iEt.retrieveUtxos(history, offset, limit) + getAddress(tree)(history) match { + case Some(iEa) => iEa.retrieveUtxos(history, offset, limit) case None => Seq.empty[IndexedErgoBox] } } @@ -223,5 +226,15 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } + private def getTokenInfoById(id: ModifierId): Future[Option[IndexedToken]] = { + getHistory.map { history => + history.typedModifierById[IndexedToken](id) + } + } + + private def getTokenInfoByIdR: Route = (get & pathPrefix("box" / "byId") & modifierId) { id => + ApiResponse(getTokenInfoById(id)) + } + } diff --git a/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala b/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala index 1b54b8bbff..2261a6a046 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala @@ -3,7 +3,7 @@ package org.ergoplatform.modifiers.history import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionSerializer} import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} -import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoAddressSerializer, IndexedErgoBox, IndexedErgoBoxSerializer, IndexedErgoTransaction, IndexedErgoTransactionSerializer, IndexedErgoTree, IndexedErgoTreeSerializer, NumericBoxIndex, NumericBoxIndexSerializer, NumericTxIndex, NumericTxIndexSerializer} +import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoAddressSerializer, IndexedErgoBox, IndexedErgoBoxSerializer, IndexedErgoTransaction, IndexedErgoTransactionSerializer, NumericBoxIndex, NumericBoxIndexSerializer, NumericTxIndex, NumericTxIndexSerializer} import scorex.core.serialization.ScorexSerializer import scorex.util.serialization.{Reader, Writer} @@ -23,9 +23,6 @@ object HistoryModifierSerializer extends ScorexSerializer[BlockSection] { case m: Extension => w.put(Extension.modifierTypeId) ExtensionSerializer.serialize(m, w) - case m: IndexedErgoTree => - w.put(IndexedErgoTree.modifierTypeId) - IndexedErgoTreeSerializer.serialize(m, w) case m: IndexedErgoAddress => w.put(IndexedErgoAddress.modifierTypeId) IndexedErgoAddressSerializer.serialize(m, w) @@ -56,8 +53,6 @@ object HistoryModifierSerializer extends ScorexSerializer[BlockSection] { BlockTransactionsSerializer.parse(r) case Extension.`modifierTypeId` => ExtensionSerializer.parse(r) - case IndexedErgoTree.`modifierTypeId` => - IndexedErgoTreeSerializer.parse(r) case IndexedErgoAddress.`modifierTypeId` => IndexedErgoAddressSerializer.parse(r) case IndexedErgoTransaction.`modifierTypeId` => diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index a09ccc9bc4..d0ba167b5a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -1,6 +1,7 @@ package org.ergoplatform.nodeView.history.extra import akka.actor.{Actor, ActorRef, ActorSystem, Props} +import org.ergoplatform.ErgoBox.TokenId import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, Input} import org.ergoplatform.modifiers.history.BlockTransactions @@ -35,20 +36,21 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private val saveLimit: Int = cacheSettings.history.extraCacheSize * 10 - private var done: Boolean = false + private var caughtUp: Boolean = false private def chainHeight: Int = _history.fullBlockHeight private var _history: ErgoHistory = null - private def history: ErgoHistoryReader = _history.asInstanceOf[ErgoHistoryReader] + private def history: ErgoHistoryReader = _history.getReader private def historyStorage: HistoryStorage = _history.historyStorage // fast access private val general: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] private val boxes: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] - private val trees: ArrayBuffer[IndexedErgoTree] = ArrayBuffer.empty[IndexedErgoTree] private val addresses: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] + private val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] + private def findBoxOpt(id: Array[Byte]): Option[Int] = { cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first if(java.util.Arrays.equals(boxes(i).serializedId, id)) return Some(i) @@ -56,31 +58,20 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) None } - private def findTreeOpt(id: Array[Byte]): Option[Int] = { - cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(java.util.Arrays.equals(trees(i).serializedId, id)) return Some(i) - } - None - } - - private def findAddressOpt(id: Array[Byte]): Option[Int] = { + private def findAddressOpt(id: ModifierId): Option[Int] = { cfor(addresses.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(java.util.Arrays.equals(addresses(i).serializedId, id)) return Some(i) + if(addresses(i).treeHash == id) return Some(i) } None } - private def modCount: Int = general.length + boxes.length + trees.length + addresses.length + private def modCount: Int = general.length + boxes.length + addresses.length private def saveProgress(): Unit = { val start: Long = System.nanoTime() // perform segmentation on big modifiers - val treesLen: Int = trees.length - cfor(0)(_ < treesLen, _ + 1) { i => - if(trees(i).boxes.length > segmentTreshold) trees += trees(i).splitToSegment() - } val addressesLen: Int = addresses.length cfor(0)(_ < addressesLen, _ + 1) { i => if(addresses(i).txs.length > segmentTreshold || (addresses(i).boxes.length > segmentTreshold)) addresses ++= addresses(i).splitToSegment() @@ -88,11 +79,10 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // merge all modifiers to an Array, avoids reallocations durin concatenation (++) val all: Array[BlockSection] = new Array[BlockSection](modCount) - val offset: Array[Int] = Array(0, general.length, general.length + boxes.length, general.length + boxes.length + trees.length) + val offset: Array[Int] = Array(0, general.length, general.length + boxes.length) cfor(0)(_ < general.length , _ + 1) { i => all(i + offset(0)) = general(i) } cfor(0)(_ < boxes.length , _ + 1) { i => all(i + offset(1)) = boxes(i) } - cfor(0)(_ < trees.length , _ + 1) { i => all(i + offset(2)) = trees(i) } - cfor(0)(_ < addresses.length, _ + 1) { i => all(i + offset(3)) = addresses(i) } + cfor(0)(_ < addresses.length, _ + 1) { i => all(i + offset(2)) = addresses(i) } // insert modifiers and progress info to db indexedHeightBuffer.clear() @@ -104,12 +94,11 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) val end: Long = System.nanoTime() - log.info(s"Processed ${boxes.length} boxes, ${trees.length} ErgoTrees, ${addresses.length} addresses and inserted them to database in ${(end - start) / 1000000000D}s") + log.info(s"Processed ${addresses.length} addresses with ${boxes.length} boxes and inserted them to database in ${(end - start) / 1000000D}ms") // clear buffers for next batch general.clear() boxes.clear() - trees.clear() addresses.clear() } @@ -124,15 +113,21 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) general += IndexedErgoTransaction(tx.id, indexedHeight, globalTxIndex) general += NumericTxIndex(globalTxIndex, tx.id) + tokens.clear() + //process transaction inputs if(indexedHeight != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => val input: Input = tx.inputs(i) findBoxOpt(input.boxId) match { - case Some(x) => boxes(x).asSpent(tx.id, indexedHeight) // box found in last saveLimit modifiers, update + case Some(x) => // box found in last saveLimit modifiers, update + boxes(x).asSpent(tx.id, indexedHeight) + tokens ++= boxes(x).box.additionalTokens.toArray case None => // box not found in last saveLimit blocks history.typedModifierById[IndexedErgoBox](bytesToId(input.boxId)) match { - case Some(x) => boxes += x.asSpent(tx.id, indexedHeight) // box found in DB, update + case Some(x) => // box found in DB, update + boxes += x.asSpent(tx.id, indexedHeight) + tokens ++= x.box.additionalTokens.toArray case None => log.warn(s"Input for box ${bytesToId(input.boxId)} not found in database") // box not found at all (this shouldn't happen) } } @@ -142,32 +137,25 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) //process transaction outputs cfor(0)(_ < tx.outputs.size, _ + 1) { i => val box: ErgoBox = tx.outputs(i) - boxes += new IndexedErgoBox(Some(indexedHeight), None, None, box, globalBoxIndex, Some(chainHeight - indexedHeight)) // box by id + boxes += new IndexedErgoBox(Some(indexedHeight), None, None, box, globalBoxIndex) // box by id general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number - // box by ergotree - val treeHash: Array[Byte] = IndexedErgoTreeSerializer.ergoTreeHash(box.ergoTree) - findTreeOpt(treeHash) match { - case Some(x) => trees(x).addBox(globalBoxIndex) // ergotree found in last saveLimit modifiers, update - case None => // ergotree not found in last saveLimit blocks - history.typedModifierById[IndexedErgoTree](bytesToId(treeHash)) match { - case Some(x) => trees += x.addBox(globalBoxIndex) // ergotree found in DB, update - case None => trees += IndexedErgoTree(bytesToId(treeHash), ListBuffer(globalBoxIndex)) // ergotree not found at all, record - } - } - // box by address - val addrHash: Array[Byte] = IndexedErgoAddressSerializer.hashAddress(IndexedErgoBoxSerializer.getAddress(box.ergoTree)) - findAddressOpt(addrHash) match { + val treeHash: ModifierId = bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)) + findAddressOpt(treeHash) match { case Some(x) => addresses(x).addTx(globalTxIndex).addBox(globalBoxIndex) // address found in last saveLimit modifiers, update case None => // address not found in last saveLimit blocks - history.typedModifierById[IndexedErgoAddress](bytesToId(addrHash)) match { + history.typedModifierById[IndexedErgoAddress](treeHash) match { case Some(x) => addresses += x.addTx(globalTxIndex).addBox(globalBoxIndex)//address found in DB, update - case None => addresses += IndexedErgoAddress(bytesToId(addrHash), ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) //address not found at all, record + case None => addresses += IndexedErgoAddress(treeHash, ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) //address not found at all, record } } - // TODO token indexing + if(IndexedTokenSerializer.tokenRegistersSet(box)) + cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => + if(!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) + general += IndexedTokenSerializer.fromBox(box) + } globalBoxIndex += 1 @@ -179,9 +167,14 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) log.info(s"Buffered block #$indexedHeight / $chainHeight [txs: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] (buffer: $modCount / $saveLimit)") - if(done) indexedHeight = height // update height here after caught up with chain + if(caughtUp) { + + indexedHeight = height // update height here after caught up with chain + + if(modCount >= saveLimit || // modifier limit reached to write to db + history.fullBlockHeight == history.headersHeight) saveProgress() // write to db every block after caught up - if(modCount >= saveLimit || done) saveProgress() // save index every saveLimit modifiers or every block after caught up + }else if(modCount >= saveLimit) saveProgress() // active syncing, write to db after modifier limit } @@ -198,7 +191,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) index(history.bestBlockTransactionsAt(indexedHeight).get, indexedHeight) } - done = true + caughtUp = true log.info("Indexer caught up with chain") } @@ -213,7 +206,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) override def receive: Receive = { case SemanticallySuccessfulModifier(fb: ErgoFullBlock) => - if(done) // after the indexer caught up with the chain, stay up to date + if(caughtUp) // after the indexer caught up with the chain, stay up to date index(fb.blockTransactions, fb.height) case Start(history: ErgoHistory) => _history = history @@ -245,7 +238,7 @@ object ExtraIndexerRef { } // faster id to bytes - no safety checks - def fastIdToBytes(id: ModifierId): Array[Byte] = { + private[extra] def fastIdToBytes(id: ModifierId): Array[Byte] = { val x: Array[Byte] = Array.ofDim[Byte](id.length / 2) cfor(0)(_ < x.length, _ + 2) {i => x(i / 2) = ((hexIndex(id(i)) << 4) | hexIndex(id(i + 1))).toByte} x diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 6016d47c5c..c778e0d823 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -1,7 +1,6 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoAddress -import org.ergoplatform.ErgoAddressEncoder.{ChecksumLength, hash256} import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes @@ -12,16 +11,17 @@ import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} +import sigmastate.Values.ErgoTree import scala.collection.mutable.ListBuffer import spire.syntax.all.cfor -case class IndexedErgoAddress(addressHash: ModifierId, +case class IndexedErgoAddress(treeHash: ModifierId, txs: ListBuffer[Long], boxes: ListBuffer[Long]) extends BlockSection { override val sizeOpt: Option[Int] = None - override def serializedId: Array[Byte] = fastIdToBytes(addressHash) + override def serializedId: Array[Byte] = fastIdToBytes(treeHash) override def parentId: ModifierId = null override val modifierTypeId: ModifierTypeId = IndexedErgoAddress.modifierTypeId override type M = IndexedErgoAddress @@ -37,7 +37,7 @@ case class IndexedErgoAddress(addressHash: ModifierId, if (offset > txs.length) { val range: Array[Int] = getSegmentsForRange(offset, limit) cfor(0)(_ < range.length, _ + 1) { i => - txs ++=: history.typedModifierById[IndexedErgoAddress](txSegmentId(addressHash, txSegmentCount - range(i))).get.txs + txs ++=: history.typedModifierById[IndexedErgoAddress](txSegmentId(treeHash, txSegmentCount - range(i))).get.txs } } slice(txs, offset, limit).map(n => NumericTxIndex.getTxByNumber(history, n).get).toArray @@ -47,7 +47,7 @@ case class IndexedErgoAddress(addressHash: ModifierId, if(offset > boxes.length) { val range: Array[Int] = getSegmentsForRange(offset, limit) cfor(0)(_ < range.length, _ + 1) { i => - boxes ++=: history.typedModifierById[IndexedErgoAddress](boxSegmentId(addressHash, boxSegmentCount - range(i))).get.boxes + boxes ++=: history.typedModifierById[IndexedErgoAddress](boxSegmentId(treeHash, boxSegmentCount - range(i))).get.boxes } } slice(boxes, offset, limit).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray @@ -59,7 +59,7 @@ case class IndexedErgoAddress(addressHash: ModifierId, var segment = boxSegmentCount while(data.length < limit && segment > 0) { segment -= 1 - data ++=: history.typedModifierById[IndexedErgoAddress](boxSegmentId(addressHash, segment)).get.boxes + data ++=: history.typedModifierById[IndexedErgoAddress](boxSegmentId(treeHash, segment)).get.boxes .map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) } slice(data, offset, limit).toArray @@ -79,12 +79,12 @@ case class IndexedErgoAddress(addressHash: ModifierId, require(segmentTreshold < txs.length || segmentTreshold < boxes.length, "address does not have enough transactions or boxes for segmentation") val data: ListBuffer[IndexedErgoAddress] = ListBuffer.empty[IndexedErgoAddress] if(segmentTreshold < txs.length) { - data += new IndexedErgoAddress(txSegmentId(addressHash, txSegmentCount), txs.take(segmentTreshold), ListBuffer.empty[Long]) + data += new IndexedErgoAddress(txSegmentId(treeHash, txSegmentCount), txs.take(segmentTreshold), ListBuffer.empty[Long]) txSegmentCount += 1 txs.remove(0, segmentTreshold) } if(segmentTreshold < boxes.length) { - data += new IndexedErgoAddress(boxSegmentId(addressHash, segmentTreshold), ListBuffer.empty[Long], boxes.take(segmentTreshold)) + data += new IndexedErgoAddress(boxSegmentId(treeHash, segmentTreshold), ListBuffer.empty[Long], boxes.take(segmentTreshold)) boxSegmentCount += 1 boxes.remove(0, segmentTreshold) } @@ -94,20 +94,15 @@ case class IndexedErgoAddress(addressHash: ModifierId, object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] { - def addressToBytes(address: ErgoAddress): Array[Byte] = { - val withNetworkByte = (ExtraIndexerRef.getAddressEncoder.networkPrefix + address.addressTypePrefix).toByte +: address.contentBytes - withNetworkByte ++ hash256(withNetworkByte).take(ChecksumLength) - } - - def hashAddress(address: ErgoAddress): Array[Byte] = Algos.hash(addressToBytes(address)) + def hashAddress(address: ErgoAddress): Array[Byte] = Algos.hash(address.script.bytes) - def addressToModifierId(address: ErgoAddress): ModifierId = bytesToId(hashAddress(address)) + def hashErgoTree(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) def boxSegmentId(addressHash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(addressHash + " box segment " + segmentNum)) def txSegmentId(addressHash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(addressHash + " tx segment " + segmentNum)) override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = { - w.putBytes(fastIdToBytes(iEa.addressHash)) + w.putBytes(fastIdToBytes(iEa.treeHash)) w.putUInt(iEa.txs.length) cfor(0)(_ < iEa.txs.length, _ + 1) { i => w.putLong(iEa.txs(i))} w.putUInt(iEa.boxes.length) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala index ad0c76d86a..a41c7ae5e3 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -16,8 +16,7 @@ class IndexedErgoBox(val inclusionHeightOpt: Option[Int], var spendingTxIdOpt: Option[ModifierId], var spendingHeightOpt: Option[Int], val box: ErgoBox, - val globalIndex: Long, - val confirmations: Option[Int]) + val globalIndex: Long) extends WalletBox(TrackedBox(box.transactionId, box.index, inclusionHeightOpt, @@ -25,7 +24,7 @@ class IndexedErgoBox(val inclusionHeightOpt: Option[Int], spendingHeightOpt, box, Set.empty[ScanId]), - confirmations) with BlockSection { + None) with BlockSection { override def parentId: ModifierId = null override val modifierTypeId: ModifierTypeId = IndexedErgoBox.modifierTypeId @@ -56,7 +55,6 @@ object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { w.putOption[Int](iEb.spendingHeightOpt)(_.putInt(_)) ErgoBoxSerializer.serialize(iEb.box, w) w.putLong(iEb.globalIndex) - w.putOption[Int](iEb.confirmations)(_.putInt(_)) } override def parse(r: Reader): IndexedErgoBox = { @@ -65,8 +63,7 @@ object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { val spendingHeightOpt: Option[Int] = r.getOption[Int](r.getInt()) val box: ErgoBox = ErgoBoxSerializer.parse(r) val globalIndex: Long = r.getLong() - val confirmations: Option[Int] = r.getOption[Int](r.getInt()) - new IndexedErgoBox(inclusionHeightOpt, spendingTxIdOpt, spendingHeightOpt, box, globalIndex, confirmations) + new IndexedErgoBox(inclusionHeightOpt, spendingTxIdOpt, spendingHeightOpt, box, globalIndex) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala deleted file mode 100644 index 2932f26002..0000000000 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTree.scala +++ /dev/null @@ -1,95 +0,0 @@ -package org.ergoplatform.nodeView.history.extra - -import org.ergoplatform.modifiers.BlockSection -import org.ergoplatform.nodeView.history.ErgoHistoryReader -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes -import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getSegmentsForRange, segmentTreshold, slice} -import org.ergoplatform.nodeView.history.extra.IndexedErgoTreeSerializer.segmentId -import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId -import scorex.core.serialization.ScorexSerializer -import scorex.util.{ModifierId, bytesToId} -import scorex.util.serialization.{Reader, Writer} -import sigmastate.Values.ErgoTree - -import scala.collection.mutable.ListBuffer -import spire.syntax.all.cfor - -case class IndexedErgoTree(treeHash: ModifierId, boxes: ListBuffer[Long]) extends BlockSection { - override val sizeOpt: Option[Int] = None - override def serializedId: Array[Byte] = fastIdToBytes(treeHash) - override def parentId: ModifierId = null - override val modifierTypeId: ModifierTypeId = IndexedErgoTree.modifierTypeId - override type M = IndexedErgoTree - override def serializer: ScorexSerializer[IndexedErgoTree] = IndexedErgoTreeSerializer - - private[extra] var segmentCount: Int = 0 - - def boxCount(): Long = segmentTreshold * segmentCount + boxes.length - - def retrieveBoxes(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = { - if(offset > boxes.length) { - val range: Array[Int] = getSegmentsForRange(offset, limit) - cfor(0)(_ < range.length, _ + 1) { i => - boxes ++=: history.typedModifierById[IndexedErgoAddress](segmentId(treeHash, segmentCount - range(i))).get.boxes - } - } - slice(boxes, offset, limit).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray - } - - def retrieveUtxos(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = { - val data: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] - data ++= boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) - var segment = segmentCount - while(data.length < limit && segment > 0) { - segment -= 1 - data ++=: history.typedModifierById[IndexedErgoAddress](segmentId(treeHash, segment)).get.boxes - .map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) - } - slice(data, offset, limit).toArray - } - - - def addBox(num: Long): IndexedErgoTree = { - boxes += num - this - } - - def splitToSegment(): IndexedErgoTree = { - require(segmentTreshold < boxes.length, "ergotree does not have enough boxes for segmentation") - val iEt: IndexedErgoTree = IndexedErgoTree(segmentId(treeHash, segmentCount), boxes.take(segmentTreshold)) - segmentCount += 1 - boxes.remove(0, segmentTreshold) - iEt - } -} - -object IndexedErgoTreeSerializer extends ScorexSerializer[IndexedErgoTree] { - - def ergoTreeHash(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) - - def segmentId(treeHash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(treeHash + " segment " + segmentNum)) - - override def serialize(iEt: IndexedErgoTree, w: Writer): Unit = { - w.putBytes(iEt.serializedId) - w.putLong(iEt.boxes.length) - cfor(0)(_ < iEt.boxes.length, _ + 1) { i => w.putLong(iEt.boxes(i))} - w.putInt(iEt.segmentCount) - } - - override def parse(r: Reader): IndexedErgoTree = { - val treeHash: ModifierId = bytesToId(r.getBytes(32)) - val boxesLen: Long = r.getLong() - val boxes: ListBuffer[Long] = ListBuffer.empty[Long] - cfor(0)(_ < boxesLen, _ + 1) { _ => boxes += r.getLong()} - val iEt: IndexedErgoTree = IndexedErgoTree(treeHash, boxes) - iEt.segmentCount = r.getInt() - iEt - } -} - -object IndexedErgoTree { - - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 20.toByte - -} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 7761f94b53..45357328cf 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -9,13 +9,14 @@ import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} import sigmastate.SByte -import sigmastate.Values.CollectionConstant +import sigmastate.Values.ConcreteCollection case class IndexedToken(tokenId: ModifierId, + boxId: ModifierId, amount: Long, name: String, description: String, - decimals: Byte) extends BlockSection { + decimals: Int) extends BlockSection { override def parentId: ModifierId = null override val modifierTypeId: ModifierTypeId = IndexedToken.modifierTypeId override type M = IndexedToken @@ -26,33 +27,36 @@ case class IndexedToken(tokenId: ModifierId, object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { - def hasTokenEmission(box: ErgoBox): Boolean = - box.additionalTokens.size >= 1 && box.additionalRegisters.contains(R4) && box.additionalRegisters.contains(R5) && box.additionalRegisters.contains(R6) + def tokenRegistersSet(box: ErgoBox): Boolean = + box.additionalRegisters.contains(R4) && box.additionalRegisters.contains(R5) && box.additionalRegisters.contains(R6) def fromBox(box: ErgoBox): IndexedToken = IndexedToken(bytesToId(box.additionalTokens(0)._1), + bytesToId(box.id), box.additionalTokens(0)._2, - new String(box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]].items.asInstanceOf[Seq[Byte]].toArray, "UTF-8"), - new String(box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]].items.asInstanceOf[Seq[Byte]].toArray, "UTF-8"), - box.additionalRegisters(R6).asInstanceOf[CollectionConstant[SByte.type]].items.asInstanceOf[Seq[Byte]].head) + new String(box.additionalRegisters(R4).asInstanceOf[ConcreteCollection[SByte.type]].items.map(_.asInstanceOf[Byte]).toArray, "UTF-8"), + new String(box.additionalRegisters(R5).asInstanceOf[ConcreteCollection[SByte.type]].items.map(_.asInstanceOf[Byte]).toArray, "UTF-8"), + new String(box.additionalRegisters(R6).asInstanceOf[ConcreteCollection[SByte.type]].items.map(_.asInstanceOf[Byte]).toArray, "UTF-8").toInt) override def serialize(iT: IndexedToken, w: Writer): Unit = { w.putBytes(iT.serializedId) + w.putBytes(fastIdToBytes(iT.boxId)) w.putULong(iT.amount) w.putUShort(iT.name.length) w.putBytes(iT.name.getBytes("UTF-8")) w.putUShort(iT.description.length) w.putBytes(iT.description.getBytes("UTF-8")) - w.putUByte(iT.decimals) + w.putInt(iT.decimals) } override def parse(r: Reader): IndexedToken = { val tokenId: ModifierId = bytesToId(r.getBytes(32)) + val boxId: ModifierId = bytesToId(r.getBytes(32)) val amount: Long = r.getULong() val name: String = new String(r.getBytes(r.getUShort), "UTF-8") val description: String = new String(r.getBytes(r.getUShort), "UTF-8") - val decimals: Byte = r.getUByte().toByte - IndexedToken(tokenId, amount, name, description, decimals) + val decimals: Int = r.getInt() + IndexedToken(tokenId, boxId, amount, name, description, decimals) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 3e092cbf81..c304ff49a2 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -4,7 +4,7 @@ import com.google.common.cache.CacheBuilder import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.modifiers.history.HistoryModifierSerializer import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoTree} +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress import org.ergoplatform.settings.{Algos, CacheSettings, ErgoSettings} import scorex.core.utils.ScorexEncoding import scorex.db.{ByteArrayWrapper, LDBFactory, LDBKVStore} @@ -44,7 +44,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e private def cacheModifier(mod: BlockSection): Unit = mod.modifierTypeId match { case Header.modifierTypeId => headersCache.put(mod.id, mod) - case IndexedErgoAddress.modifierTypeId | IndexedErgoTree.modifierTypeId => extraCache.put(mod.id, mod) + case IndexedErgoAddress.modifierTypeId => extraCache.put(mod.id, mod) // only cache "big" modifiers case _ => blockSectionsCache.put(mod.id, mod) } From 0ca8f1d161a1f6dc4d45d3e21a0064d800d8dcd7 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Tue, 30 Aug 2022 20:39:15 +0200 Subject: [PATCH 11/90] tested and fixed token tracking --- .../history/HistoryModifierSerializer.scala | 7 ++++- .../nodeView/history/extra/ExtraIndexer.scala | 28 +++++++++---------- .../nodeView/history/extra/IndexedToken.scala | 23 ++++++++++----- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala b/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala index 2261a6a046..d132eccadb 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala @@ -3,7 +3,7 @@ package org.ergoplatform.modifiers.history import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionSerializer} import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} -import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoAddressSerializer, IndexedErgoBox, IndexedErgoBoxSerializer, IndexedErgoTransaction, IndexedErgoTransactionSerializer, NumericBoxIndex, NumericBoxIndexSerializer, NumericTxIndex, NumericTxIndexSerializer} +import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoAddressSerializer, IndexedErgoBox, IndexedErgoBoxSerializer, IndexedErgoTransaction, IndexedErgoTransactionSerializer, IndexedToken, IndexedTokenSerializer, NumericBoxIndex, NumericBoxIndexSerializer, NumericTxIndex, NumericTxIndexSerializer} import scorex.core.serialization.ScorexSerializer import scorex.util.serialization.{Reader, Writer} @@ -38,6 +38,9 @@ object HistoryModifierSerializer extends ScorexSerializer[BlockSection] { case m: NumericBoxIndex => w.put(NumericBoxIndex.modifierTypeId) NumericBoxIndexSerializer.serialize(m, w) + case m: IndexedToken => + w.put(IndexedToken.modifierTypeId) + IndexedTokenSerializer.serialize(m, w) case m => throw new Error(s"Serialization for unknown modifier: $m") } @@ -63,6 +66,8 @@ object HistoryModifierSerializer extends ScorexSerializer[BlockSection] { NumericTxIndexSerializer.parse(r) case NumericBoxIndex.`modifierTypeId` => NumericBoxIndexSerializer.parse(r) + case IndexedToken.`modifierTypeId` => + IndexedTokenSerializer.parse(r) case m => throw new Error(s"Deserialization for unknown type byte: $m") } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index d0ba167b5a..4b0668f519 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -47,7 +47,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // fast access private val general: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] private val boxes: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] - private val addresses: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] + private val trees: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] private val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] @@ -58,23 +58,23 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) None } - private def findAddressOpt(id: ModifierId): Option[Int] = { - cfor(addresses.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(addresses(i).treeHash == id) return Some(i) + private def findTreeOpt(id: ModifierId): Option[Int] = { + cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first + if(trees(i).treeHash == id) return Some(i) } None } - private def modCount: Int = general.length + boxes.length + addresses.length + private def modCount: Int = general.length + boxes.length + trees.length private def saveProgress(): Unit = { val start: Long = System.nanoTime() // perform segmentation on big modifiers - val addressesLen: Int = addresses.length + val addressesLen: Int = trees.length cfor(0)(_ < addressesLen, _ + 1) { i => - if(addresses(i).txs.length > segmentTreshold || (addresses(i).boxes.length > segmentTreshold)) addresses ++= addresses(i).splitToSegment() + if(trees(i).txs.length > segmentTreshold || trees(i).boxes.length > segmentTreshold) trees ++= trees(i).splitToSegment() } // merge all modifiers to an Array, avoids reallocations durin concatenation (++) @@ -82,7 +82,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) val offset: Array[Int] = Array(0, general.length, general.length + boxes.length) cfor(0)(_ < general.length , _ + 1) { i => all(i + offset(0)) = general(i) } cfor(0)(_ < boxes.length , _ + 1) { i => all(i + offset(1)) = boxes(i) } - cfor(0)(_ < addresses.length, _ + 1) { i => all(i + offset(2)) = addresses(i) } + cfor(0)(_ < trees.length, _ + 1) { i => all(i + offset(2)) = trees(i) } // insert modifiers and progress info to db indexedHeightBuffer.clear() @@ -94,12 +94,12 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) val end: Long = System.nanoTime() - log.info(s"Processed ${addresses.length} addresses with ${boxes.length} boxes and inserted them to database in ${(end - start) / 1000000D}ms") + log.info(s"Processed ${trees.length} ErgoTrees with ${boxes.length} boxes and inserted them to database in ${(end - start) / 1000000D}ms") // clear buffers for next batch general.clear() boxes.clear() - addresses.clear() + trees.clear() } @@ -142,12 +142,12 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // box by address val treeHash: ModifierId = bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)) - findAddressOpt(treeHash) match { - case Some(x) => addresses(x).addTx(globalTxIndex).addBox(globalBoxIndex) // address found in last saveLimit modifiers, update + findTreeOpt(treeHash) match { + case Some(x) => trees(x).addTx(globalTxIndex).addBox(globalBoxIndex) // address found in last saveLimit modifiers, update case None => // address not found in last saveLimit blocks history.typedModifierById[IndexedErgoAddress](treeHash) match { - case Some(x) => addresses += x.addTx(globalTxIndex).addBox(globalBoxIndex)//address found in DB, update - case None => addresses += IndexedErgoAddress(treeHash, ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) //address not found at all, record + case Some(x) => trees += x.addTx(globalTxIndex).addBox(globalBoxIndex)//address found in DB, update + case None => trees += IndexedErgoAddress(treeHash, ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) //address not found at all, record } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 45357328cf..c60a23123d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -6,10 +6,11 @@ import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer -import scorex.util.{ModifierId, bytesToId} +import scorex.util.{ModifierId, ScorexLogging, bytesToId} import scorex.util.serialization.{Reader, Writer} -import sigmastate.SByte -import sigmastate.Values.ConcreteCollection +import sigmastate.{SByte, SType} +import sigmastate.Values.CollectionConstant +import special.collection.Coll case class IndexedToken(tokenId: ModifierId, boxId: ModifierId, @@ -25,18 +26,26 @@ case class IndexedToken(tokenId: ModifierId, override def serializedId: Array[Byte] = fastIdToBytes(tokenId) } -object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { +object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] with ScorexLogging { def tokenRegistersSet(box: ErgoBox): Boolean = box.additionalRegisters.contains(R4) && box.additionalRegisters.contains(R5) && box.additionalRegisters.contains(R6) + def getDecimals(x: SType#WrappedType): Int = { + try { + x.asInstanceOf[Coll[Byte]].toArray(0) + }catch { + case _: Throwable => x.asInstanceOf[Int] + } + } + def fromBox(box: ErgoBox): IndexedToken = IndexedToken(bytesToId(box.additionalTokens(0)._1), bytesToId(box.id), box.additionalTokens(0)._2, - new String(box.additionalRegisters(R4).asInstanceOf[ConcreteCollection[SByte.type]].items.map(_.asInstanceOf[Byte]).toArray, "UTF-8"), - new String(box.additionalRegisters(R5).asInstanceOf[ConcreteCollection[SByte.type]].items.map(_.asInstanceOf[Byte]).toArray, "UTF-8"), - new String(box.additionalRegisters(R6).asInstanceOf[ConcreteCollection[SByte.type]].items.map(_.asInstanceOf[Byte]).toArray, "UTF-8").toInt) + new String(box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), + new String(box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), + getDecimals(box.additionalRegisters(R6).value)) override def serialize(iT: IndexedToken, w: Writer): Unit = { w.putBytes(iT.serializedId) From 519cd14e49231efb878592e7fb52cd8322a82346 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Wed, 31 Aug 2022 02:07:28 +0200 Subject: [PATCH 12/90] efficient serialization --- .../nodeView/history/extra/ExtraIndexer.scala | 27 ++++++++++--------- .../history/storage/HistoryStorage.scala | 10 +++++-- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 4b0668f519..7026e481e5 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -3,7 +3,7 @@ package org.ergoplatform.nodeView.history.extra import akka.actor.{Actor, ActorRef, ActorSystem, Props} import org.ergoplatform.ErgoBox.TokenId import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} -import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, Input} +import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier @@ -51,9 +51,9 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] - private def findBoxOpt(id: Array[Byte]): Option[Int] = { + private def findBoxOpt(id: ModifierId): Option[Int] = { cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(java.util.Arrays.equals(boxes(i).serializedId, id)) return Some(i) + if(boxes(i).id == id) return Some(i) } None } @@ -80,9 +80,9 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // merge all modifiers to an Array, avoids reallocations durin concatenation (++) val all: Array[BlockSection] = new Array[BlockSection](modCount) val offset: Array[Int] = Array(0, general.length, general.length + boxes.length) - cfor(0)(_ < general.length , _ + 1) { i => all(i + offset(0)) = general(i) } - cfor(0)(_ < boxes.length , _ + 1) { i => all(i + offset(1)) = boxes(i) } - cfor(0)(_ < trees.length, _ + 1) { i => all(i + offset(2)) = trees(i) } + cfor(0)(_ < general.length, _ + 1) { i => all(i + offset(0)) = general(i) } + cfor(0)(_ < boxes.length , _ + 1) { i => all(i + offset(1)) = boxes(i) } + cfor(0)(_ < trees.length , _ + 1) { i => all(i + offset(2)) = trees(i) } // insert modifiers and progress info to db indexedHeightBuffer.clear() @@ -118,17 +118,17 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) //process transaction inputs if(indexedHeight != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => - val input: Input = tx.inputs(i) - findBoxOpt(input.boxId) match { + val inputId: ModifierId = bytesToId(tx.inputs(i).boxId) + findBoxOpt(inputId) match { case Some(x) => // box found in last saveLimit modifiers, update boxes(x).asSpent(tx.id, indexedHeight) tokens ++= boxes(x).box.additionalTokens.toArray case None => // box not found in last saveLimit blocks - history.typedModifierById[IndexedErgoBox](bytesToId(input.boxId)) match { + history.typedModifierById[IndexedErgoBox](inputId) match { case Some(x) => // box found in DB, update boxes += x.asSpent(tx.id, indexedHeight) tokens ++= x.box.additionalTokens.toArray - case None => log.warn(s"Input for box ${bytesToId(input.boxId)} not found in database") // box not found at all (this shouldn't happen) + case None => log.warn(s"Input for box $inputId not found in database") // box not found at all (this shouldn't happen) } } } @@ -146,12 +146,13 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) case Some(x) => trees(x).addTx(globalTxIndex).addBox(globalBoxIndex) // address found in last saveLimit modifiers, update case None => // address not found in last saveLimit blocks history.typedModifierById[IndexedErgoAddress](treeHash) match { - case Some(x) => trees += x.addTx(globalTxIndex).addBox(globalBoxIndex)//address found in DB, update - case None => trees += IndexedErgoAddress(treeHash, ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) //address not found at all, record + case Some(x) => trees += x.addTx(globalTxIndex).addBox(globalBoxIndex) // address found in DB, update + case None => trees += IndexedErgoAddress(treeHash, ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) // address not found at all, record } } - if(IndexedTokenSerializer.tokenRegistersSet(box)) + // check if box is creating a new token, if yes record it + if(box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => if(!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) general += IndexedTokenSerializer.fromBox(box) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index c304ff49a2..e6dbaed1c2 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -87,10 +87,16 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e def contains(id: ModifierId): Boolean = objectsStore.get(idToBytes(id)).isDefined + def serializeModifiers(mods: Array[BlockSection]): Array[(Array[Byte], Array[Byte])] = { + val arr: Array[(Array[Byte], Array[Byte])] = new Array[(Array[Byte], Array[Byte])](mods.length) + cfor(0)(_ < arr.length, _ + 1) { i => arr(i) = (mods(i).serializedId, HistoryModifierSerializer.toBytes(mods(i))) } + arr + } + def insert(indexesToInsert: Array[(ByteArrayWrapper, Array[Byte])], objectsToInsert: Array[BlockSection]): Try[Unit] = { objectsStore.insert( - objectsToInsert.map(m => m.serializedId -> HistoryModifierSerializer.toBytes(m)) + serializeModifiers(objectsToInsert) ).flatMap { _ => cfor(0)(_ < objectsToInsert.length, _ + 1) { i => cacheModifier(objectsToInsert(i))} if (indexesToInsert.nonEmpty) { @@ -104,7 +110,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e def insertExtra(indexesToInsert: Array[(Array[Byte], Array[Byte])], objectsToInsert: Array[BlockSection]): Unit = { - extraStore.insert(objectsToInsert.map(m => m.serializedId -> HistoryModifierSerializer.toBytes(m))) + extraStore.insert(serializeModifiers(objectsToInsert)) cfor(0)(_ < objectsToInsert.length, _ + 1) { i => cacheModifier(objectsToInsert(i))} cfor(0)(_ < indexesToInsert.length, _ + 1) { i => extraStore.insert(indexesToInsert(i)._1, indexesToInsert(i)._2)} } From 49752c88db0cd5f51402e1f5febe7ebdaf10cc95 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Wed, 31 Aug 2022 23:46:58 +0200 Subject: [PATCH 13/90] small fixes, token detection broken --- src/main/resources/api/openapi.yaml | 69 +++++++++++++++++++ .../http/api/BlockchainApiRoute.scala | 4 +- .../nodeView/history/extra/ExtraIndexer.scala | 4 +- .../history/extra/IndexedErgoAddress.scala | 2 +- .../nodeView/history/extra/IndexedToken.scala | 16 +++-- 5 files changed, 85 insertions(+), 10 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 45cef3bdac..9c9be0fa4d 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -256,6 +256,42 @@ components: minimum: 0 example: 83927 + IndexedToken: + type: object + description: Token indexed with extra information + required: + -id + -boxId + -emissionAmount + -name + -description + -decimals + properties: + id: + description: Id of the token + $ref: '#/components/schemas/ModifierId' + boxId: + description: Id of the box that created the token + $ref: '#/components/schemas/ModifierId' + emissionAmount: + description: The total supply of the token + type: integer + format: int64 + minimum: 1 + example: 3500000 + name: + description: The name of the token + type: string + description: + description: The description of the token + type: string + decimals: + description: The number of decimals the token supports + type: integer + format: int32 + minimum: 0 + example: 8 + UnsignedErgoTransaction: type: object description: Unsigned Ergo transaction @@ -5652,6 +5688,39 @@ paths: application/json: schema: $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/token/byId/{tokenId}: + get: + summary: Retrieve + operationId: getTokenById + tags: + - blockchain + parameters: + - in: path + name: tokenId + required: true + description: id of the wanted token + schema: + type: string + responses: + '200': + description: token with wanted id + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedToken' + '404': + description: No token found with wanted id + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' default: description: Error content: diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 597b473238..c202c09156 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -68,7 +68,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } private def getTxByIdR: Route = (get & pathPrefix("transaction" / "byId") & modifierId) { id => - ApiResponse(getTxByIdF(bytesToId(Base16.decode(id).get))) + ApiResponse(getTxByIdF(id)) } private def getTxByIndex(index: Long): Future[Option[IndexedErgoTransaction]] = @@ -232,7 +232,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getTokenInfoByIdR: Route = (get & pathPrefix("box" / "byId") & modifierId) { id => + private def getTokenInfoByIdR: Route = (get & pathPrefix("token" / "byId") & modifierId) { id => ApiResponse(getTokenInfoById(id)) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 7026e481e5..d95a61f270 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -240,8 +240,8 @@ object ExtraIndexerRef { // faster id to bytes - no safety checks private[extra] def fastIdToBytes(id: ModifierId): Array[Byte] = { - val x: Array[Byte] = Array.ofDim[Byte](id.length / 2) - cfor(0)(_ < x.length, _ + 2) {i => x(i / 2) = ((hexIndex(id(i)) << 4) | hexIndex(id(i + 1))).toByte} + val x: Array[Byte] = new Array[Byte](id.length / 2) + cfor(0)(_ < id.length, _ + 2) {i => x(i / 2) = ((hexIndex(id(i)) << 4) | hexIndex(id(i + 1))).toByte} x } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index c778e0d823..1214596166 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -66,7 +66,7 @@ case class IndexedErgoAddress(treeHash: ModifierId, } def addTx(tx: Long): IndexedErgoAddress = { - if(!txs.takeRight(5).contains(tx)) txs += tx // check for duplicates + if(txs(txs.length - 1) != tx) txs += tx // check for duplicates this } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index c60a23123d..a6166c60f4 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -28,12 +28,18 @@ case class IndexedToken(tokenId: ModifierId, object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] with ScorexLogging { - def tokenRegistersSet(box: ErgoBox): Boolean = - box.additionalRegisters.contains(R4) && box.additionalRegisters.contains(R5) && box.additionalRegisters.contains(R6) + // Stop type erasure + trait ByteColl extends Coll[Byte] + trait SColl extends CollectionConstant[SByte.type] + + def tokenRegistersSet(box: ErgoBox): Boolean = // these don't work + box.additionalRegisters.contains(R4) && box.additionalRegisters(R4).isInstanceOf[SColl] && + box.additionalRegisters.contains(R5) && box.additionalRegisters(R5).isInstanceOf[SColl] && + box.additionalRegisters.contains(R6) && (box.additionalRegisters(R6).value.isInstanceOf[ByteColl] || box.additionalRegisters(R6).value.isInstanceOf[Int]) def getDecimals(x: SType#WrappedType): Int = { try { - x.asInstanceOf[Coll[Byte]].toArray(0) + x.asInstanceOf[ByteColl].toArray(0) }catch { case _: Throwable => x.asInstanceOf[Int] } @@ -43,8 +49,8 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] with Scorex IndexedToken(bytesToId(box.additionalTokens(0)._1), bytesToId(box.id), box.additionalTokens(0)._2, - new String(box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), - new String(box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), + new String(box.additionalRegisters(R4).asInstanceOf[SColl].value.toArray, "UTF-8"), + new String(box.additionalRegisters(R5).asInstanceOf[SColl].value.toArray, "UTF-8"), getDecimals(box.additionalRegisters(R6).value)) override def serialize(iT: IndexedToken, w: Writer): Unit = { From 5cefc3561e418d89ec90849f58d1c9eaec12968e Mon Sep 17 00:00:00 2001 From: jellymlg Date: Fri, 2 Sep 2022 06:14:39 +0200 Subject: [PATCH 14/90] fixed token indexing --- .../http/api/BlockchainApiRoute.scala | 3 +- .../nodeView/history/extra/ExtraIndexer.scala | 19 ++++++----- .../nodeView/history/extra/IndexedToken.scala | 33 ++++++++++++++----- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index c202c09156..4e1690fe6a 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -122,7 +122,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } private def getBoxByIdR: Route = (get & pathPrefix("box" / "byId") & modifierId) { id => - ApiResponse(getBoxByIdF(bytesToId(Base16.decode(id).get))) + ApiResponse(getBoxByIdF(id)) } private def getBoxByIndex(index: Long): Future[Option[IndexedErgoBox]] = @@ -237,4 +237,3 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index d95a61f270..95bb98bcdf 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -48,8 +48,9 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private val general: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] private val boxes: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] private val trees: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] + private val tokens: ArrayBuffer[IndexedToken] = ArrayBuffer.empty[IndexedToken] - private val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] + private val inputTokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] private def findBoxOpt(id: ModifierId): Option[Int] = { cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first @@ -65,7 +66,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) None } - private def modCount: Int = general.length + boxes.length + trees.length + private def modCount: Int = general.length + boxes.length + trees.length + tokens.length private def saveProgress(): Unit = { @@ -79,10 +80,11 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // merge all modifiers to an Array, avoids reallocations durin concatenation (++) val all: Array[BlockSection] = new Array[BlockSection](modCount) - val offset: Array[Int] = Array(0, general.length, general.length + boxes.length) + val offset: Array[Int] = Array(0, general.length, general.length + boxes.length, general.length + boxes.length + trees.length) cfor(0)(_ < general.length, _ + 1) { i => all(i + offset(0)) = general(i) } cfor(0)(_ < boxes.length , _ + 1) { i => all(i + offset(1)) = boxes(i) } cfor(0)(_ < trees.length , _ + 1) { i => all(i + offset(2)) = trees(i) } + cfor(0)(_ < tokens.length , _ + 1) { i => all(i + offset(3)) = tokens(i) } // insert modifiers and progress info to db indexedHeightBuffer.clear() @@ -100,7 +102,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) general.clear() boxes.clear() trees.clear() - + tokens.clear() } private def index(bt: BlockTransactions, height: Int): Unit = { @@ -122,12 +124,12 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) findBoxOpt(inputId) match { case Some(x) => // box found in last saveLimit modifiers, update boxes(x).asSpent(tx.id, indexedHeight) - tokens ++= boxes(x).box.additionalTokens.toArray + inputTokens ++= boxes(x).box.additionalTokens.toArray case None => // box not found in last saveLimit blocks history.typedModifierById[IndexedErgoBox](inputId) match { case Some(x) => // box found in DB, update boxes += x.asSpent(tx.id, indexedHeight) - tokens ++= x.box.additionalTokens.toArray + inputTokens ++= x.box.additionalTokens.toArray case None => log.warn(s"Input for box $inputId not found in database") // box not found at all (this shouldn't happen) } } @@ -154,8 +156,9 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // check if box is creating a new token, if yes record it if(box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => - if(!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) - general += IndexedTokenSerializer.fromBox(box) + if(!inputTokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) { + tokens += IndexedTokenSerializer.fromBox(box) + } } globalBoxIndex += 1 diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index a6166c60f4..1261287a7d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -8,8 +8,8 @@ import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, ScorexLogging, bytesToId} import scorex.util.serialization.{Reader, Writer} -import sigmastate.{SByte, SType} import sigmastate.Values.CollectionConstant +import sigmastate.{SByte, SType} import special.collection.Coll case class IndexedToken(tokenId: ModifierId, @@ -28,14 +28,29 @@ case class IndexedToken(tokenId: ModifierId, object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] with ScorexLogging { + def tokenRegistersSet(box: ErgoBox): Boolean = { + + // registers exist + if(!box.additionalRegisters.contains(R4) || + !box.additionalRegisters.contains(R5) || + !box.additionalRegisters.contains(R6)) + return false + + // registers correct type + try { + box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]] + box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]] + getDecimals(box.additionalRegisters(R6).value) + }catch { + case _: Throwable => return false + } + + // ok + true + } + // Stop type erasure trait ByteColl extends Coll[Byte] - trait SColl extends CollectionConstant[SByte.type] - - def tokenRegistersSet(box: ErgoBox): Boolean = // these don't work - box.additionalRegisters.contains(R4) && box.additionalRegisters(R4).isInstanceOf[SColl] && - box.additionalRegisters.contains(R5) && box.additionalRegisters(R5).isInstanceOf[SColl] && - box.additionalRegisters.contains(R6) && (box.additionalRegisters(R6).value.isInstanceOf[ByteColl] || box.additionalRegisters(R6).value.isInstanceOf[Int]) def getDecimals(x: SType#WrappedType): Int = { try { @@ -49,8 +64,8 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] with Scorex IndexedToken(bytesToId(box.additionalTokens(0)._1), bytesToId(box.id), box.additionalTokens(0)._2, - new String(box.additionalRegisters(R4).asInstanceOf[SColl].value.toArray, "UTF-8"), - new String(box.additionalRegisters(R5).asInstanceOf[SColl].value.toArray, "UTF-8"), + new String(box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), + new String(box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), getDecimals(box.additionalRegisters(R6).value)) override def serialize(iT: IndexedToken, w: Writer): Unit = { From 8a84786a88979d3500600a64461cd1bbb4cdbf63 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Fri, 2 Sep 2022 19:23:29 +0200 Subject: [PATCH 15/90] cfor not necessary for serialization --- src/main/resources/api/openapi.yaml | 2 +- .../nodeView/history/storage/HistoryStorage.scala | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 2546260731..db6c49bfc8 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -5697,7 +5697,7 @@ paths: /blockchain/token/byId/{tokenId}: get: - summary: Retrieve + summary: Retrieve minting information about a token operationId: getTokenById tags: - blockchain diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index e6dbaed1c2..2031555a45 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -87,16 +87,10 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e def contains(id: ModifierId): Boolean = objectsStore.get(idToBytes(id)).isDefined - def serializeModifiers(mods: Array[BlockSection]): Array[(Array[Byte], Array[Byte])] = { - val arr: Array[(Array[Byte], Array[Byte])] = new Array[(Array[Byte], Array[Byte])](mods.length) - cfor(0)(_ < arr.length, _ + 1) { i => arr(i) = (mods(i).serializedId, HistoryModifierSerializer.toBytes(mods(i))) } - arr - } - def insert(indexesToInsert: Array[(ByteArrayWrapper, Array[Byte])], objectsToInsert: Array[BlockSection]): Try[Unit] = { objectsStore.insert( - serializeModifiers(objectsToInsert) + objectsToInsert.map(mod => (mod.serializedId, HistoryModifierSerializer.toBytes(mod))) ).flatMap { _ => cfor(0)(_ < objectsToInsert.length, _ + 1) { i => cacheModifier(objectsToInsert(i))} if (indexesToInsert.nonEmpty) { @@ -110,7 +104,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e def insertExtra(indexesToInsert: Array[(Array[Byte], Array[Byte])], objectsToInsert: Array[BlockSection]): Unit = { - extraStore.insert(serializeModifiers(objectsToInsert)) + extraStore.insert(objectsToInsert.map(mod => (mod.serializedId, HistoryModifierSerializer.toBytes(mod)))) cfor(0)(_ < objectsToInsert.length, _ + 1) { i => cacheModifier(objectsToInsert(i))} cfor(0)(_ < indexesToInsert.length, _ + 1) { i => extraStore.insert(indexesToInsert(i)._1, indexesToInsert(i)._2)} } From 8e1935abd55e2c3dc794c525cba42972c31b6919 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sat, 3 Sep 2022 00:35:07 +0200 Subject: [PATCH 16/90] bugfixes, token indexing done --- .../http/api/BlockchainApiRoute.scala | 24 ++++++------------- .../http/api/ErgoBaseApiRoute.scala | 12 ++++++++++ .../nodeView/history/extra/ExtraIndexer.scala | 4 +++- .../history/extra/IndexedErgoAddress.scala | 8 +++---- .../history/storage/HistoryStorage.scala | 2 +- 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 4e1690fe6a..59a985965f 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -17,7 +17,6 @@ import sigmastate.Values.ErgoTree import sigmastate.serialization.ErgoTreeSerializer import scala.concurrent.Future -import scala.util.{Failure, Success} case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings, extraIndexerOpt: Option[ActorRef]) (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute with ApiCodecs { @@ -28,7 +27,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private val MaxItems = 16384 - val addressEncoder: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder + override val ae: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder override val route: Route = pathPrefix("blockchain") { @@ -88,14 +87,11 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getTxsByAddressR: Route = (get & pathPrefix("transaction" / "byAddress") & path(Segment) & paging) { (address, offset, limit) => + private def getTxsByAddressR: Route = (get & pathPrefix("transaction" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => if(limit > MaxItems) { BadRequest(s"No more than $MaxItems transactions can be requested") }else { - addressEncoder.fromString(address) match { - case Success(addr) => ApiResponse(getTxsByAddress(addr, offset, limit)) - case Failure(_) => BadRequest("Incorrect address format") - } + ApiResponse(getTxsByAddress(address, offset, limit)) } } @@ -142,14 +138,11 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByAddressR: Route = (get & pathPrefix("box" / "byAddress") & path(Segment) & paging) { (address, offset, limit) => + private def getBoxesByAddressR: Route = (get & pathPrefix("box" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => if(limit > MaxItems) { BadRequest(s"No more than $MaxItems boxes can be requested") }else { - addressEncoder.fromString(address) match { - case Success(addr) => ApiResponse(getBoxesByAddress(addr, offset, limit)) - case Failure(_) => BadRequest("Incorrect address format") - } + ApiResponse(getBoxesByAddress(address, offset, limit)) } } @@ -161,14 +154,11 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByAddressUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byAddress") & path(Segment) & paging) { (address, offset, limit) => + private def getBoxesByAddressUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => if(limit > MaxItems) { BadRequest(s"No more than $MaxItems boxes can be requested") }else { - addressEncoder.fromString(address) match { - case Success(addr) => ApiResponse(getBoxesByAddressUnspent(addr, offset, limit)) - case Failure(_) => BadRequest("Incorrect address format") - } + ApiResponse(getBoxesByAddressUnspent(address, offset, limit)) } } diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala index 97949231c4..3a189be78e 100644 --- a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala @@ -11,6 +11,7 @@ import org.ergoplatform.settings.{Algos, ErgoSettings} import scorex.core.api.http.{ApiError, ApiRoute} import scorex.util.{ModifierId, bytesToId} import akka.pattern.ask +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedTransaction import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome._ @@ -22,6 +23,8 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { implicit val ec: ExecutionContextExecutor = context.dispatcher + implicit val ae: ErgoAddressEncoder = null + val modifierId: Directive1[ModifierId] = pathPrefix(Segment).flatMap(handleModifierId) val modifierIdGet: Directive1[ModifierId] = parameters("id".as[String]) @@ -34,6 +37,15 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { } } + val ergoAddress: Directive1[ErgoAddress] = pathPrefix(Segment).flatMap(handleErgoAddress) + + private def handleErgoAddress(value: String): Directive1[ErgoAddress] = { + ae.fromString(value) match { + case Success(addr) => provide(addr) + case _ => reject(ValidationRejection("Wrong address format")) + } + } + private def getStateAndPool(readersHolder: ActorRef): Future[(ErgoStateReader, ErgoMemPoolReader)] = { (readersHolder ? GetReaders).mapTo[Readers].map { rs => (rs.s, rs.m) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 95bb98bcdf..b91c87f271 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -107,6 +107,8 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private def index(bt: BlockTransactions, height: Int): Unit = { + if(height < indexedHeight) return // do not process older blocks again after caught up (actor message queue) + cfor(0)(_ < bt.txs.size, _ + 1) { n => val tx: ErgoTransaction = bt.txs(n) @@ -115,7 +117,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) general += IndexedErgoTransaction(tx.id, indexedHeight, globalTxIndex) general += NumericTxIndex(globalTxIndex, tx.id) - tokens.clear() + inputTokens.clear() //process transaction inputs if(indexedHeight != 1) { //only after 1st block (skip genesis box) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 1214596166..385e80fc5d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -66,7 +66,7 @@ case class IndexedErgoAddress(treeHash: ModifierId, } def addTx(tx: Long): IndexedErgoAddress = { - if(txs(txs.length - 1) != tx) txs += tx // check for duplicates + if(txs.last != tx) txs += tx // check for duplicates this } @@ -84,7 +84,7 @@ case class IndexedErgoAddress(treeHash: ModifierId, txs.remove(0, segmentTreshold) } if(segmentTreshold < boxes.length) { - data += new IndexedErgoAddress(boxSegmentId(treeHash, segmentTreshold), ListBuffer.empty[Long], boxes.take(segmentTreshold)) + data += new IndexedErgoAddress(boxSegmentId(treeHash, boxSegmentCount), ListBuffer.empty[Long], boxes.take(segmentTreshold)) boxSegmentCount += 1 boxes.remove(0, segmentTreshold) } @@ -98,8 +98,8 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] def hashErgoTree(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) - def boxSegmentId(addressHash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(addressHash + " box segment " + segmentNum)) - def txSegmentId(addressHash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(addressHash + " tx segment " + segmentNum)) + def boxSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " box segment " + segmentNum)) + def txSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " tx segment " + segmentNum)) override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = { w.putBytes(fastIdToBytes(iEa.treeHash)) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 2031555a45..7b40cf94a6 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -85,7 +85,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e def get(id: ModifierId): Option[Array[Byte]] = objectsStore.get(idToBytes(id)).orElse(extraStore.get(idToBytes(id))) - def contains(id: ModifierId): Boolean = objectsStore.get(idToBytes(id)).isDefined + def contains(id: ModifierId): Boolean = get(id).isDefined def insert(indexesToInsert: Array[(ByteArrayWrapper, Array[Byte])], objectsToInsert: Array[BlockSection]): Try[Unit] = { From 65d9f621a9de8e6ca370603207e5973d863510cd Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sat, 3 Sep 2022 19:41:49 +0200 Subject: [PATCH 17/90] major bugfixes in token indexing --- .../org/ergoplatform/http/api/ApiCodecs.scala | 2 +- .../http/api/BlockchainApiRoute.scala | 32 +++++++------------ .../http/api/ErgoBaseApiRoute.scala | 13 ++++++++ .../nodeView/history/extra/ExtraIndexer.scala | 9 ++---- .../nodeView/history/extra/IndexedToken.scala | 29 +++++++++-------- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index bf9266322b..933c2e69ed 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -496,7 +496,7 @@ trait ApiCodecs extends JsonCodecs { implicit val IndexedTokenEncoder: Encoder[IndexedToken] = { token => Json.obj( - "id" -> token.id.asJson, + "id" -> token.tokenId.asJson, "boxId" -> token.boxId.asJson, "emissionAmount" -> token.amount.asJson, "name" -> token.name.asJson, diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 59a985965f..a18a05dacc 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -11,10 +11,8 @@ import org.ergoplatform.settings.ErgoSettings import scorex.core.api.http.ApiError.BadRequest import scorex.core.api.http.ApiResponse import scorex.core.settings.RESTApiSettings -import scorex.util.encode.Base16 import scorex.util.{ModifierId, bytesToId} import sigmastate.Values.ErgoTree -import sigmastate.serialization.ErgoTreeSerializer import scala.concurrent.Future @@ -184,15 +182,11 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByErgoTreeR: Route = (get & pathPrefix("box" / "byErgoTree") & path(Segment) & paging) { (tree, offset, limit) => - try { - if(limit > MaxItems) { - BadRequest(s"No more than $MaxItems boxes can be requested") - }else { - ApiResponse(getBoxesByErgoTree(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(Base16.decode(tree).get), offset, limit)) - } - }catch { - case e: Exception => BadRequest(s"${e.getMessage}") + private def getBoxesByErgoTreeR: Route = (get & pathPrefix("box" / "byErgoTree") & ergoTree & paging) { (tree, offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + ApiResponse(getBoxesByErgoTree(tree, offset, limit)) } } @@ -204,21 +198,17 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByErgoTreeUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byErgoTree") & path(Segment) & paging) { (tree, offset, limit) => - try { - if(limit > MaxItems) { - BadRequest(s"No more than $MaxItems boxes can be requested") - }else { - ApiResponse(getBoxesByErgoTreeUnspent(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(Base16.decode(tree).get), offset, limit)) - } - }catch { - case e: Exception => BadRequest(s"${e.getMessage}") + private def getBoxesByErgoTreeUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byErgoTree") & ergoTree & paging) { (tree, offset, limit) => + if(limit > MaxItems) { + BadRequest(s"No more than $MaxItems boxes can be requested") + }else { + ApiResponse(getBoxesByErgoTreeUnspent(tree, offset, limit)) } } private def getTokenInfoById(id: ModifierId): Future[Option[IndexedToken]] = { getHistory.map { history => - history.typedModifierById[IndexedToken](id) + history.typedModifierById[IndexedToken](IndexedTokenSerializer.uniqueId(id)) } } diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala index 3a189be78e..8625441fbb 100644 --- a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala @@ -15,6 +15,9 @@ import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedTransaction import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome._ +import scorex.util.encode.Base16 +import sigmastate.Values.ErgoTree +import sigmastate.serialization.ErgoTreeSerializer import scala.concurrent.{ExecutionContextExecutor, Future} import scala.util.{Success, Try} @@ -46,6 +49,16 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { } } + val ergoTree: Directive1[ErgoTree] = pathPrefix(Segment).flatMap(handleErgoTree) + + private def handleErgoTree(value: String): Directive1[ErgoTree] = { + Base16.decode(value) match { + case Success(bytes) => provide(ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(bytes)) + case _ => reject(ValidationRejection("Invalid hex data")) + } + + } + private def getStateAndPool(readersHolder: ActorRef): Future[(ErgoStateReader, ErgoMemPoolReader)] = { (readersHolder ? GetReaders).mapTo[Readers].map { rs => (rs.s, rs.m) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index b91c87f271..c88e89bed6 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -48,7 +48,6 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private val general: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] private val boxes: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] private val trees: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] - private val tokens: ArrayBuffer[IndexedToken] = ArrayBuffer.empty[IndexedToken] private val inputTokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] @@ -66,7 +65,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) None } - private def modCount: Int = general.length + boxes.length + trees.length + tokens.length + private def modCount: Int = general.length + boxes.length + trees.length private def saveProgress(): Unit = { @@ -80,11 +79,10 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // merge all modifiers to an Array, avoids reallocations durin concatenation (++) val all: Array[BlockSection] = new Array[BlockSection](modCount) - val offset: Array[Int] = Array(0, general.length, general.length + boxes.length, general.length + boxes.length + trees.length) + val offset: Array[Int] = Array(0, general.length, general.length + boxes.length) cfor(0)(_ < general.length, _ + 1) { i => all(i + offset(0)) = general(i) } cfor(0)(_ < boxes.length , _ + 1) { i => all(i + offset(1)) = boxes(i) } cfor(0)(_ < trees.length , _ + 1) { i => all(i + offset(2)) = trees(i) } - cfor(0)(_ < tokens.length , _ + 1) { i => all(i + offset(3)) = tokens(i) } // insert modifiers and progress info to db indexedHeightBuffer.clear() @@ -102,7 +100,6 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) general.clear() boxes.clear() trees.clear() - tokens.clear() } private def index(bt: BlockTransactions, height: Int): Unit = { @@ -159,7 +156,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) if(box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => if(!inputTokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) { - tokens += IndexedTokenSerializer.fromBox(box) + general += IndexedTokenSerializer.fromBox(box) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 1261287a7d..1df2895e9b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -4,13 +4,14 @@ import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.{R4, R5, R6} import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes +import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.uniqueId +import org.ergoplatform.settings.Algos import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer -import scorex.util.{ModifierId, ScorexLogging, bytesToId} +import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} -import sigmastate.Values.CollectionConstant +import sigmastate.Values.{CollectionConstant, EvaluatedValue} import sigmastate.{SByte, SType} -import special.collection.Coll case class IndexedToken(tokenId: ModifierId, boxId: ModifierId, @@ -23,10 +24,13 @@ case class IndexedToken(tokenId: ModifierId, override type M = IndexedToken override def serializer: ScorexSerializer[IndexedToken] = IndexedTokenSerializer override val sizeOpt: Option[Int] = None - override def serializedId: Array[Byte] = fastIdToBytes(tokenId) + override def serializedId: Array[Byte] = fastIdToBytes(uniqueId(tokenId)) } -object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] with ScorexLogging { +object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { + + // necessary, because token ids are sometimes identical to box ids, which causes overwrites + def uniqueId(tokenId: ModifierId): ModifierId = bytesToId(Algos.hash(tokenId + "token")) def tokenRegistersSet(box: ErgoBox): Boolean = { @@ -40,7 +44,7 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] with Scorex try { box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]] box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]] - getDecimals(box.additionalRegisters(R6).value) + getDecimals(box.additionalRegisters(R6)) }catch { case _: Throwable => return false } @@ -49,14 +53,11 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] with Scorex true } - // Stop type erasure - trait ByteColl extends Coll[Byte] - - def getDecimals(x: SType#WrappedType): Int = { + def getDecimals(reg: EvaluatedValue[_ <: SType]): Int = { try { - x.asInstanceOf[ByteColl].toArray(0) + new String(reg.asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8").toInt }catch { - case _: Throwable => x.asInstanceOf[Int] + case _: Throwable => reg.value.asInstanceOf[Int] } } @@ -66,10 +67,10 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] with Scorex box.additionalTokens(0)._2, new String(box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), new String(box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), - getDecimals(box.additionalRegisters(R6).value)) + getDecimals(box.additionalRegisters(R6))) override def serialize(iT: IndexedToken, w: Writer): Unit = { - w.putBytes(iT.serializedId) + w.putBytes(fastIdToBytes(iT.tokenId)) w.putBytes(fastIdToBytes(iT.boxId)) w.putULong(iT.amount) w.putUShort(iT.name.length) From 4d1eeeb324ffa0e0acac4aaf662c82dc4c74f745 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sat, 3 Sep 2022 21:17:53 +0200 Subject: [PATCH 18/90] minor changes --- src/main/scala/org/ergoplatform/ErgoApp.scala | 2 +- .../org/ergoplatform/http/api/BlockchainApiRoute.scala | 2 +- .../nodeView/history/extra/ExtraIndexer.scala | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index 23b11c2537..9dd55699df 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -132,7 +132,7 @@ class ErgoApp(args: Args) extends ScorexLogging { private val apiRoutes: Seq[ApiRoute] = Seq( EmissionApiRoute(ergoSettings), ErgoUtilsApiRoute(ergoSettings), - BlockchainApiRoute(readersHolderRef, ergoSettings, indexerRefOpt), + BlockchainApiRoute(readersHolderRef, ergoSettings), ErgoPeersApiRoute(peerManagerRef, networkControllerRef, syncTracker, deliveryTracker, scorexSettings.restApi), InfoApiRoute(statsCollectorRef, scorexSettings.restApi, timeProvider), BlocksApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings), diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index a18a05dacc..776a67aaf8 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -16,7 +16,7 @@ import sigmastate.Values.ErgoTree import scala.concurrent.Future -case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings, extraIndexerOpt: Option[ActorRef]) +case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings) (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute with ApiCodecs { val settings: RESTApiSettings = ergoSettings.scorexSettings.restApi diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index c88e89bed6..fcb8fc3aae 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -124,12 +124,12 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) case Some(x) => // box found in last saveLimit modifiers, update boxes(x).asSpent(tx.id, indexedHeight) inputTokens ++= boxes(x).box.additionalTokens.toArray - case None => // box not found in last saveLimit blocks + case None => // box not found in last saveLimit modifiers history.typedModifierById[IndexedErgoBox](inputId) match { case Some(x) => // box found in DB, update boxes += x.asSpent(tx.id, indexedHeight) inputTokens ++= x.box.additionalTokens.toArray - case None => log.warn(s"Input for box $inputId not found in database") // box not found at all (this shouldn't happen) + case None => log.warn(s"Unknown box used as input: $inputId") // box not found at all (this shouldn't happen) } } } @@ -145,7 +145,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) val treeHash: ModifierId = bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)) findTreeOpt(treeHash) match { case Some(x) => trees(x).addTx(globalTxIndex).addBox(globalBoxIndex) // address found in last saveLimit modifiers, update - case None => // address not found in last saveLimit blocks + case None => // address not found in last saveLimit modifiers history.typedModifierById[IndexedErgoAddress](treeHash) match { case Some(x) => trees += x.addTx(globalTxIndex).addBox(globalBoxIndex) // address found in DB, update case None => trees += IndexedErgoAddress(treeHash, ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) // address not found at all, record @@ -260,10 +260,10 @@ object ExtraIndexerRefHolder { private var _actor: Option[ActorRef] = None private var running: Boolean = false - def start(history: ErgoHistory): Unit = if(_actor.isDefined && !running) { + private[history] def start(history: ErgoHistory): Unit = if(_actor.isDefined && !running) { _actor.get ! Start(history) running = true } - protected[extra] def init(actor: ActorRef): Unit = _actor = Some(actor) + private[extra] def init(actor: ActorRef): Unit = _actor = Some(actor) } From 23bba4f64364110a656384e474061a0edd36b51d Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sat, 3 Sep 2022 22:47:22 +0200 Subject: [PATCH 19/90] negative decimals fixed --- .../org/ergoplatform/nodeView/history/extra/IndexedToken.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 1df2895e9b..3b7a340f54 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -67,7 +67,7 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { box.additionalTokens(0)._2, new String(box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), new String(box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), - getDecimals(box.additionalRegisters(R6))) + math.max(getDecimals(box.additionalRegisters(R6)), 0)) override def serialize(iT: IndexedToken, w: Writer): Unit = { w.putBytes(fastIdToBytes(iT.tokenId)) From 6f9f450cf07eba0f7920747b10354bb08f0ce92f Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sat, 3 Sep 2022 23:22:01 +0200 Subject: [PATCH 20/90] bugfixes --- src/main/scala/org/ergoplatform/ErgoApp.scala | 8 +++----- .../nodeView/history/extra/ExtraIndexer.scala | 10 +++++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index 9dd55699df..7b19b4e0dc 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -102,11 +102,9 @@ class ErgoApp(args: Args) extends ScorexLogging { } // Create an instance of ExtraIndexer actor if "extraIndex = true" in config - private val indexerRefOpt: Option[ActorRef] = - if(ergoSettings.nodeSettings.extraIndex) - Some(ExtraIndexerRef(ergoSettings.chainSettings, ergoSettings.cacheSettings)) - else - None + if(ergoSettings.nodeSettings.extraIndex) + Some(ExtraIndexerRef(ergoSettings.chainSettings, ergoSettings.cacheSettings)) + ExtraIndexerRef.setAddressEncoder(ergoSettings.addressEncoder) // initialize an accessible address encoder regardless of extra indexing being enabled private val syncTracker = ErgoSyncTracker(scorexSettings.network, timeProvider) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index fcb8fc3aae..986a962ede 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -34,6 +34,8 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private var globalBoxIndex: Long = 0L private val globalBoxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) + private var lastWroteToDB: Int = 0 + private val saveLimit: Int = cacheSettings.history.extraCacheSize * 10 private var caughtUp: Boolean = false @@ -100,11 +102,13 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) general.clear() boxes.clear() trees.clear() + + lastWroteToDB = indexedHeight } private def index(bt: BlockTransactions, height: Int): Unit = { - if(height < indexedHeight) return // do not process older blocks again after caught up (actor message queue) + if(caughtUp && height <= indexedHeight) return // do not process older blocks again after caught up (due to actor message queue) cfor(0)(_ < bt.txs.size, _ + 1) { n => @@ -168,7 +172,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } - log.info(s"Buffered block #$indexedHeight / $chainHeight [txs: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] (buffer: $modCount / $saveLimit)") + log.info(s"Buffered block #$height / $chainHeight [txs: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] (buffer: $modCount / $saveLimit)") if(caughtUp) { @@ -204,7 +208,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } override def postStop(): Unit = { - log.info(s"Stopped extra indexer at height ${indexedHeight - 1}") + log.info(s"Stopped extra indexer at height $lastWroteToDB") } override def receive: Receive = { From 7a8e379977bdeaef7dd9bc2dff2212af2288a778 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 4 Sep 2022 23:32:03 +0200 Subject: [PATCH 21/90] refractors and more bugfixes --- src/main/resources/api/openapi.yaml | 54 ++++++++-------- src/main/scala/org/ergoplatform/ErgoApp.scala | 2 +- .../org/ergoplatform/http/api/ApiCodecs.scala | 5 +- .../http/api/BlockchainApiRoute.scala | 64 +++++++++++++------ .../nodeView/history/extra/ExtraIndexer.scala | 42 ++++++------ .../history/extra/IndexedErgoAddress.scala | 35 +++++----- .../extra/IndexedErgoTransaction.scala | 33 +++++----- .../nodeView/history/extra/NumericIndex.scala | 4 -- .../history/storage/HistoryStorage.scala | 2 +- 9 files changed, 135 insertions(+), 106 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index db6c49bfc8..1e22ccf94b 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -5312,7 +5312,7 @@ paths: description: amount of elements to skip from the start schema: type: integer - format: int64 + format: int32 default: 0 - in: query name: limit @@ -5320,8 +5320,8 @@ paths: description: amount of elements to retrieve schema: type: integer - format: int64 - default: 10 + format: int32 + default: 5 responses: '200': description: transactions associated with wanted address @@ -5358,7 +5358,7 @@ paths: description: amount of elements to skip from the start schema: type: integer - format: int64 + format: int32 default: 0 - in: query name: limit @@ -5366,18 +5366,18 @@ paths: description: amount of elements to retrieve schema: type: integer - format: int64 - default: 10 + format: int32 + default: 5 responses: '200': - description: transactions in wanted range + description: transactions ids in wanted range content: application/json: schema: type: array - description: Array of transactions + description: Array of transaction ids items: - $ref: '#/components/schemas/IndexedErgoTransaction' + $ref: '#/components/schemas/ModifierId' default: description: Error content: @@ -5470,7 +5470,7 @@ paths: description: amount of elements to skip from the start schema: type: integer - format: int64 + format: int32 default: 0 - in: query name: limit @@ -5478,8 +5478,8 @@ paths: description: amount of elements to retrieve schema: type: integer - format: int64 - default: 10 + format: int32 + default: 5 responses: '200': description: boxes associated with wanted address @@ -5522,7 +5522,7 @@ paths: description: amount of elements to skip from the start schema: type: integer - format: int64 + format: int32 default: 0 - in: query name: limit @@ -5530,8 +5530,8 @@ paths: description: amount of elements to retrieve schema: type: integer - format: int64 - default: 10 + format: int32 + default: 5 responses: '200': description: unspent boxes associated with wanted address @@ -5568,7 +5568,7 @@ paths: description: amount of elements to skip from the start schema: type: integer - format: int64 + format: int32 default: 0 - in: query name: limit @@ -5576,18 +5576,18 @@ paths: description: amount of elements to retrieve schema: type: integer - format: int64 - default: 10 + format: int32 + default: 5 responses: '200': - description: boxes in wanted range + description: box ids in wanted range content: application/json: schema: type: array - description: Array of boxes + description: Array of box ids items: - $ref: '#/components/schemas/IndexedErgoBox' + $ref: '#/components/schemas/ModifierId' default: description: Error content: @@ -5615,7 +5615,7 @@ paths: description: amount of elements to skip from the start schema: type: integer - format: int64 + format: int32 default: 0 - in: query name: limit @@ -5623,8 +5623,8 @@ paths: description: amount of elements to retrieve schema: type: integer - format: int64 - default: 10 + format: int32 + default: 5 responses: '200': description: boxes with wanted ergotree @@ -5665,7 +5665,7 @@ paths: description: amount of elements to skip from the start schema: type: integer - format: int64 + format: int32 default: 0 - in: query name: limit @@ -5673,8 +5673,8 @@ paths: description: amount of elements to retrieve schema: type: integer - format: int64 - default: 10 + format: int32 + default: 5 responses: '200': description: unspent boxes with wanted ergotree diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index 7b19b4e0dc..ff6c901d6e 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -103,7 +103,7 @@ class ErgoApp(args: Args) extends ScorexLogging { // Create an instance of ExtraIndexer actor if "extraIndex = true" in config if(ergoSettings.nodeSettings.extraIndex) - Some(ExtraIndexerRef(ergoSettings.chainSettings, ergoSettings.cacheSettings)) + ExtraIndexerRef(ergoSettings.chainSettings, ergoSettings.cacheSettings) ExtraIndexerRef.setAddressEncoder(ergoSettings.addressEncoder) // initialize an accessible address encoder regardless of extra indexing being enabled diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index 933c2e69ed..c4445e4ac5 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -481,7 +481,7 @@ trait ApiCodecs extends JsonCodecs { implicit val indexedTxEncoder: Encoder[IndexedErgoTransaction] = { tx => Json.obj( - "id" -> tx.id.asJson, + "id" -> tx.txid.asJson, "blockId" -> tx.blockId.asJson, "inclusionHeight" -> tx.inclusionHeight.asJson, "timestamp" -> tx.timestamp.asJson, @@ -491,7 +491,8 @@ trait ApiCodecs extends JsonCodecs { "inputs" -> tx.inputs.asJson, "dataInputs" -> tx.dataInputs.asJson, "outputs" -> tx.outputs.asJson, - "size" -> tx.txSize.asJson) + "size" -> tx.txSize.asJson + ) } implicit val IndexedTokenEncoder: Encoder[IndexedToken] = { token => diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 776a67aaf8..afd1dfbf4e 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -6,6 +6,7 @@ import akka.pattern.ask import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} import org.ergoplatform.nodeView.ErgoReadersHolder.GetDataFromHistory import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{GlobalBoxIndexKey, GlobalTxIndexKey} import org.ergoplatform.nodeView.history.extra._ import org.ergoplatform.settings.ErgoSettings import scorex.core.api.http.ApiError.BadRequest @@ -13,7 +14,9 @@ import scorex.core.api.http.ApiResponse import scorex.core.settings.RESTApiSettings import scorex.util.{ModifierId, bytesToId} import sigmastate.Values.ErgoTree +import spire.implicits.cfor +import java.nio.ByteBuffer import scala.concurrent.Future case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings) @@ -21,7 +24,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting val settings: RESTApiSettings = ergoSettings.scorexSettings.restApi - val paging: Directive[(Long, Long)] = parameters("offset".as[Long] ? 0L, "limit".as[Long] ? 10L) + val paging: Directive[(Int, Int)] = parameters("offset".as[Int] ? 0, "limit".as[Int] ? 5) private val MaxItems = 16384 @@ -58,7 +61,6 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting case None => None } - private def getTxByIdF(id: ModifierId) : Future[Option[IndexedErgoTransaction]] = getHistory.map { history => getTxById(id)(history) @@ -68,20 +70,26 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting ApiResponse(getTxByIdF(id)) } - private def getTxByIndex(index: Long): Future[Option[IndexedErgoTransaction]] = + private def getTxByIndex(index: Long)(history: ErgoHistoryReader): Option[IndexedErgoTransaction] = + getTxById(history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(index))).get.m)(history) + + private def getLastTx(history: ErgoHistoryReader): IndexedErgoTransaction = + getTxByIndex(ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalTxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong)(history).get + + private def getTxByIndexF(index: Long): Future[Option[IndexedErgoTransaction]] = getHistory.map { history => - getTxById(history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(index))).get.m)(history) + getTxByIndex(index)(history) } private def getTxByIndexR: Route = (pathPrefix("transaction" / "byIndex" / LongNumber) & get) { index => - ApiResponse(getTxByIndex(index)) + ApiResponse(getTxByIndexF(index)) } - private def getTxsByAddress(addr: ErgoAddress, offset: Long, limit: Long): Future[Option[Seq[IndexedErgoTransaction]]] = + private def getTxsByAddress(addr: ErgoAddress, offset: Int, limit: Int): Future[Seq[IndexedErgoTransaction]] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => Some(addr.retrieveTxs(history, offset, limit)) - case None => None + case Some(addr) => addr.retrieveTxs(history, offset, limit) + case None => Seq.empty[IndexedErgoTransaction] } } @@ -93,10 +101,14 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getTxRange(offset: Long, limit: Long): Future[Seq[ModifierId]] = + private def getTxRange(offset: Int, limit: Int): Future[Seq[ModifierId]] = getHistory.map { history => - val base: Int = (history.fullBlockHeight - offset).toInt - for(n <- (base - limit) to base) yield history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(n))).get.m + val base: Int = getLastTx(history).globalIndex.toInt - offset + val txIds: Array[ModifierId] = new Array[ModifierId](limit) + cfor(0)(_ < limit, _ + 1) { i => + txIds(i) = history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(base - limit + i))).get.m + } + txIds } private def getTxRangeR: Route = (pathPrefix("transaction" / "range") & paging) { (offset, limit) => @@ -119,16 +131,19 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting ApiResponse(getBoxByIdF(id)) } - private def getBoxByIndex(index: Long): Future[Option[IndexedErgoBox]] = + private def getBoxByIndex(index: Long)(history: ErgoHistoryReader): Option[IndexedErgoBox] = + getBoxById(history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(index))).get.m)(history) + + private def getBoxByIndexF(index: Long): Future[Option[IndexedErgoBox]] = getHistory.map { history => - getBoxById(history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(index))).get.m)(history) + getBoxByIndex(index)(history) } private def getBoxByIndexR: Route = (pathPrefix("box" / "byIndex" / LongNumber) & get) { index => - ApiResponse(getBoxByIndex(index)) + ApiResponse(getBoxByIndexF(index)) } - private def getBoxesByAddress(addr: ErgoAddress, offset: Long, limit: Long): Future[Seq[IndexedErgoBox]] = + private def getBoxesByAddress(addr: ErgoAddress, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(addr)(history) match { case Some(addr) => addr.retrieveBoxes(history, offset, limit) @@ -144,7 +159,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Long, limit: Long): Future[Seq[IndexedErgoBox]] = + private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(addr)(history) match { case Some(addr) => addr.retrieveUtxos(history, offset, limit) @@ -160,10 +175,17 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxRange(offset: Long, limit: Long): Future[Seq[ModifierId]] = + private def getLastBox(history: ErgoHistoryReader): IndexedErgoBox = + getBoxByIndex(ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalBoxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong)(history).get + + private def getBoxRange(offset: Int, limit: Int): Future[Seq[ModifierId]] = getHistory.map { history => - val base: Int = (history.fullBlockHeight - offset).toInt - for(n <- (base - limit) to base) yield history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(n))).get.m + val base: Int = getLastBox(history).globalIndex.toInt - offset + val boxIds: Array[ModifierId] = new Array[ModifierId](limit) + cfor(0)(_ < limit, _ + 1) { i => + boxIds(i) = history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(base - limit + i))).get.m + } + boxIds } private def getBoxRangeR: Route = (pathPrefix("box" / "range") & paging) { (offset, limit) => @@ -174,7 +196,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByErgoTree(tree: ErgoTree, offset: Long, limit: Long): Future[Seq[IndexedErgoBox]] = + private def getBoxesByErgoTree(tree: ErgoTree, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(tree)(history) match { case Some(iEa) => iEa.retrieveBoxes(history, offset, limit) @@ -190,7 +212,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByErgoTreeUnspent(tree: ErgoTree, offset: Long, limit: Long): Future[Seq[IndexedErgoBox]] = + private def getBoxesByErgoTreeUnspent(tree: ErgoTree, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(tree)(history) match { case Some(iEa) => iEa.retrieveUtxos(history, offset, limit) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 986a962ede..c5d1689044 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -7,6 +7,7 @@ import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{GlobalBoxIndexKey, GlobalTxIndexKey, IndexedHeightKey} import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.segmentTreshold @@ -22,15 +23,12 @@ import spire.syntax.all.cfor class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) extends Actor with ScorexLogging { - private val IndexedHeightKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("indexed height")).data private var indexedHeight: Int = 0 private val indexedHeightBuffer : ByteBuffer = ByteBuffer.allocate(4) - private val GlobalTxIndexKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("txns height")).data private var globalTxIndex: Long = 0L private val globalTxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) - private val GlobalBoxIndexKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("boxes height")).data private var globalBoxIndex: Long = 0L private val globalBoxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) @@ -51,7 +49,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private val boxes: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] private val trees: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] - private val inputTokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] + private val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] private def findBoxOpt(id: ModifierId): Option[Int] = { cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first @@ -110,29 +108,31 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) if(caughtUp && height <= indexedHeight) return // do not process older blocks again after caught up (due to actor message queue) + var boxCount: Int = 0 + cfor(0)(_ < bt.txs.size, _ + 1) { n => val tx: ErgoTransaction = bt.txs(n) //process transaction - general += IndexedErgoTransaction(tx.id, indexedHeight, globalTxIndex) + general += IndexedErgoTransaction(tx.id, height, globalTxIndex) general += NumericTxIndex(globalTxIndex, tx.id) - inputTokens.clear() + tokens.clear() //process transaction inputs - if(indexedHeight != 1) { //only after 1st block (skip genesis box) + if(height != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => val inputId: ModifierId = bytesToId(tx.inputs(i).boxId) findBoxOpt(inputId) match { case Some(x) => // box found in last saveLimit modifiers, update - boxes(x).asSpent(tx.id, indexedHeight) - inputTokens ++= boxes(x).box.additionalTokens.toArray + boxes(x).asSpent(tx.id, height) + tokens ++= boxes(x).box.additionalTokens.toArray case None => // box not found in last saveLimit modifiers history.typedModifierById[IndexedErgoBox](inputId) match { case Some(x) => // box found in DB, update - boxes += x.asSpent(tx.id, indexedHeight) - inputTokens ++= x.box.additionalTokens.toArray + boxes += x.asSpent(tx.id, height) + tokens ++= x.box.additionalTokens.toArray case None => log.warn(s"Unknown box used as input: $inputId") // box not found at all (this shouldn't happen) } } @@ -142,7 +142,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) //process transaction outputs cfor(0)(_ < tx.outputs.size, _ + 1) { i => val box: ErgoBox = tx.outputs(i) - boxes += new IndexedErgoBox(Some(indexedHeight), None, None, box, globalBoxIndex) // box by id + boxes += new IndexedErgoBox(Some(height), None, None, box, globalBoxIndex) // box by id general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number // box by address @@ -159,12 +159,13 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // check if box is creating a new token, if yes record it if(box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => - if(!inputTokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) { + if(!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) { general += IndexedTokenSerializer.fromBox(box) } } globalBoxIndex += 1 + boxCount += 1 } @@ -172,14 +173,15 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } - log.info(s"Buffered block #$height / $chainHeight [txs: ${bt.txs.size}, boxes: ${bt.txs.map(_.outputs.size).sum}] (buffer: $modCount / $saveLimit)") + log.info(s"Buffered block #$height / $chainHeight [txs: ${bt.txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") if(caughtUp) { indexedHeight = height // update height here after caught up with chain if(modCount >= saveLimit || // modifier limit reached to write to db - history.fullBlockHeight == history.headersHeight) saveProgress() // write to db every block after caught up + history.fullBlockHeight == history.headersHeight) // write to db every block after caught up + saveProgress() }else if(modCount >= saveLimit) saveProgress() // active syncing, write to db after modifier limit @@ -187,9 +189,9 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private def run(): Unit = { - indexedHeight = ByteBuffer.wrap(historyStorage.get(bytesToId(IndexedHeightKey)) .getOrElse(Array.fill[Byte](4){0})).getInt - globalTxIndex = ByteBuffer.wrap(historyStorage.get(bytesToId(GlobalTxIndexKey)) .getOrElse(Array.fill[Byte](8){0})).getLong - globalBoxIndex = ByteBuffer.wrap(historyStorage.get(bytesToId(GlobalBoxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong + indexedHeight = ByteBuffer.wrap(history.modifierBytesById(bytesToId(IndexedHeightKey)) .getOrElse(Array.fill[Byte](4){0})).getInt + globalTxIndex = ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalTxIndexKey)) .getOrElse(Array.fill[Byte](8){0})).getLong + globalBoxIndex = ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalBoxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong log.info(s"Started extra indexer at height $indexedHeight") @@ -251,6 +253,10 @@ object ExtraIndexerRef { x } + val IndexedHeightKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("indexed height")).data + val GlobalTxIndexKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("txns height")).data + val GlobalBoxIndexKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("boxes height")).data + def apply(chainSettings: ChainSettings, cacheSettings: CacheSettings)(implicit system: ActorSystem): ActorRef = { val actor = system.actorOf(Props.create(classOf[ExtraIndex], chainSettings, cacheSettings)) _ae = chainSettings.addressEncoder diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 385e80fc5d..2d7b280cf1 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -9,7 +9,7 @@ import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{box import org.ergoplatform.settings.Algos import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer -import scorex.util.{ModifierId, bytesToId} +import scorex.util.{ModifierId, ScorexLogging, bytesToId} import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.ErgoTree @@ -18,7 +18,7 @@ import spire.syntax.all.cfor case class IndexedErgoAddress(treeHash: ModifierId, txs: ListBuffer[Long], - boxes: ListBuffer[Long]) extends BlockSection { + boxes: ListBuffer[Long]) extends BlockSection with ScorexLogging { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = fastIdToBytes(treeHash) @@ -33,27 +33,33 @@ case class IndexedErgoAddress(treeHash: ModifierId, def txCount(): Long = segmentTreshold * txSegmentCount + txs.length def boxCount(): Long = segmentTreshold * boxSegmentCount + boxes.length - def retrieveTxs(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoTransaction] = { - if (offset > txs.length) { + def retrieveTxs(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoTransaction] = { + if (offset + limit > txs.length && txSegmentCount > 0) { val range: Array[Int] = getSegmentsForRange(offset, limit) cfor(0)(_ < range.length, _ + 1) { i => - txs ++=: history.typedModifierById[IndexedErgoAddress](txSegmentId(treeHash, txSegmentCount - range(i))).get.txs + txs ++=: (history.typedModifierById[IndexedErgoAddress](txSegmentId(treeHash, txSegmentCount - range(i))) match { + case Some(iEa) => iEa.txs + case None => ListBuffer.empty[Long] + }) } } - slice(txs, offset, limit).map(n => NumericTxIndex.getTxByNumber(history, n).get).toArray + slice(txs, offset, limit).map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray } - def retrieveBoxes(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = { - if(offset > boxes.length) { + def retrieveBoxes(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoBox] = { + if(offset + limit > boxes.length && boxSegmentCount > 0) { val range: Array[Int] = getSegmentsForRange(offset, limit) cfor(0)(_ < range.length, _ + 1) { i => - boxes ++=: history.typedModifierById[IndexedErgoAddress](boxSegmentId(treeHash, boxSegmentCount - range(i))).get.boxes + boxes ++=: (history.typedModifierById[IndexedErgoAddress](boxSegmentId(treeHash, boxSegmentCount - range(i))) match { + case Some(iEb) => iEb.boxes + case None => ListBuffer.empty[Long] + }) } } slice(boxes, offset, limit).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray } - def retrieveUtxos(history: ErgoHistoryReader, offset: Long, limit: Long): Array[IndexedErgoBox] = { + def retrieveUtxos(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoBox] = { val data: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] data ++= boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) var segment = boxSegmentCount @@ -94,9 +100,8 @@ case class IndexedErgoAddress(treeHash: ModifierId, object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] { - def hashAddress(address: ErgoAddress): Array[Byte] = Algos.hash(address.script.bytes) - def hashErgoTree(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) + def hashAddress(address: ErgoAddress): Array[Byte] = hashErgoTree(address.script) def boxSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " box segment " + segmentNum)) def txSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " tx segment " + segmentNum)) @@ -132,10 +137,10 @@ object IndexedErgoAddress { val segmentTreshold: Int = 512 - def getSegmentsForRange(offset: Long, limit: Long): Array[Int] = + def getSegmentsForRange(offset: Int, limit: Int): Array[Int] = (math.ceil((offset + limit) * 1F / segmentTreshold).toInt to math.ceil((offset + 1F) / segmentTreshold).toInt by -1).toArray.reverse - def slice[T](arr: Iterable[T], offset: Long, limit: Long): Iterable[T] = - arr.slice((arr.size - offset - limit).toInt, (arr.size - offset + 1).toInt) + def slice[T](arr: Iterable[T], offset: Int, limit: Int): Iterable[T] = + arr.slice(arr.size - offset - limit, arr.size - offset) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index 768c2118ba..5fc9c911a1 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -1,19 +1,19 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.modifiers.mempool.ErgoTransaction -import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} +import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.DataInput +import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import scorex.core.serialization.ScorexSerializer import scorex.core.ModifierTypeId import scorex.util.serialization.{Reader, Writer} -import scorex.util.{ModifierId, bytesToId} +import scorex.util.{ModifierId, ScorexLogging, bytesToId} case class IndexedErgoTransaction(txid: ModifierId, height: Int, - globalIndex: Long) extends BlockSection { + globalIndex: Long) extends BlockSection with ScorexLogging { override val modifierTypeId: ModifierTypeId = IndexedErgoTransaction.modifierTypeId override def serializedId: Array[Byte] = fastIdToBytes(txid) @@ -22,7 +22,7 @@ case class IndexedErgoTransaction(txid: ModifierId, override type M = IndexedErgoTransaction override def serializer: ScorexSerializer[IndexedErgoTransaction] = IndexedErgoTransactionSerializer - private var _blockId: ModifierId = _ + private var _blockId: ModifierId = ModifierId @@ "" private var _inclusionHeight: Int = 0 private var _timestamp: Header.Timestamp = 0L private var _index: Int = 0 @@ -44,19 +44,18 @@ case class IndexedErgoTransaction(txid: ModifierId, def retrieveBody(history: ErgoHistoryReader): IndexedErgoTransaction = { - val block: ErgoFullBlock = history.typedModifierById[Header](history.headerIdsAtHeight(height).head).flatMap(history.getFullBlock).get - val indexInBlock: Int = block.transactions.indices.find(block.transactions(_).id.equals(txid)).get - val tx: ErgoTransaction = block.transactions(indexInBlock) + val header: Header = history.typedModifierById[Header](history.bestHeaderIdAtHeight(height).get).get + val blockTxs: BlockTransactions = history.typedModifierById[BlockTransactions](header.transactionsId).get - _blockId = block.id - _inclusionHeight = block.height - _timestamp = block.header.timestamp - _index = indexInBlock - _numConfirmations = history.bestFullBlockOpt.get.height - block.height - _inputs = tx.inputs.map(input => history.typedModifierById[IndexedErgoBox](bytesToId(input.boxId)).get) - _dataInputs = tx.dataInputs - _outputs = tx.outputs.map(output => history.typedModifierById[IndexedErgoBox](bytesToId(output.id)).get) - _txSize = tx.size + _blockId = header.id + _inclusionHeight = height + _timestamp = header.timestamp + _index = blockTxs.txs.indices.find(blockTxs.txs(_).id == txid).get + _numConfirmations = history.bestFullBlockOpt.get.height - height + _inputs = blockTxs.txs(_index).inputs.map(input => history.typedModifierById[IndexedErgoBox](bytesToId(input.boxId)).get) + _dataInputs = blockTxs.txs(_index).dataInputs + _outputs = blockTxs.txs(_index).outputs.map(output => history.typedModifierById[IndexedErgoBox](bytesToId(output.id)).get) + _txSize = blockTxs.txs(_index).size this } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala index 5b7a5bae37..573b32c9b8 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala @@ -16,8 +16,6 @@ case class NumericTxIndex(n: Long, m: ModifierId) extends BlockSection { override val modifierTypeId: ModifierTypeId = NumericTxIndex.modifierTypeId override type M = NumericTxIndex override def serializer: ScorexSerializer[NumericTxIndex] = NumericTxIndexSerializer - - def hashValue: Array[Byte] = Array.empty[Byte] } object NumericTxIndexSerializer extends ScorexSerializer[NumericTxIndex] { @@ -49,8 +47,6 @@ case class NumericBoxIndex(n: Long, m: ModifierId) extends BlockSection { override val modifierTypeId: ModifierTypeId = NumericBoxIndex.modifierTypeId override type M = NumericBoxIndex override def serializer: ScorexSerializer[NumericBoxIndex] = NumericBoxIndexSerializer - - def hashValue: Array[Byte] = Array.empty[Byte] } object NumericBoxIndexSerializer extends ScorexSerializer[NumericBoxIndex] { diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 7b40cf94a6..d99220b88f 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -58,7 +58,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e } def modifierBytesById(id: ModifierId): Option[Array[Byte]] = { - objectsStore.get(idToBytes(id)).map(_.tail) // removing modifier type byte with .tail + objectsStore.get(idToBytes(id)).map(_.tail).orElse(extraStore.get(idToBytes(id))) // removing modifier type byte with .tail (only in objectsStore) } def modifierById(id: ModifierId): Option[BlockSection] = From b35350fb0c736095222c456d9fca705412fc18eb Mon Sep 17 00:00:00 2001 From: jellymlg Date: Fri, 9 Sep 2022 05:36:51 +0200 Subject: [PATCH 22/90] lots of bugfixes, added balance tracking [UNTESTED] --- .../org/ergoplatform/http/api/ApiCodecs.scala | 56 +++++++++++---- .../http/api/BlockchainApiRoute.scala | 34 ++++----- .../nodeView/history/extra/BalanceInfo.scala | 47 ++++++++++++ .../nodeView/history/extra/ExtraIndexer.scala | 25 +++++-- .../history/extra/IndexedErgoAddress.scala | 71 ++++++++++++------- .../extra/IndexedErgoTransaction.scala | 4 +- 6 files changed, 174 insertions(+), 63 deletions(-) create mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index c4445e4ac5..280b626e1b 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -479,19 +479,33 @@ trait ApiCodecs extends JsonCodecs { )) } - implicit val indexedTxEncoder: Encoder[IndexedErgoTransaction] = { tx => + implicit val indexedBoxSeqEncoder: Encoder[(Seq[IndexedErgoBox],Long)] = { iEbSeq => Json.obj( - "id" -> tx.txid.asJson, - "blockId" -> tx.blockId.asJson, - "inclusionHeight" -> tx.inclusionHeight.asJson, - "timestamp" -> tx.timestamp.asJson, - "index" -> tx.index.asJson, - "globalIndex" -> tx.globalIndex.asJson, - "numConfirmations" -> tx.numConfirmations.asJson, - "inputs" -> tx.inputs.asJson, - "dataInputs" -> tx.dataInputs.asJson, - "outputs" -> tx.outputs.asJson, - "size" -> tx.txSize.asJson + "items" -> iEbSeq._1.asJson, + "total" -> iEbSeq._2.asJson + ) + } + + implicit val indexedTxEncoder: Encoder[IndexedErgoTransaction] = { iEt => + Json.obj( + "id" -> iEt.txid.asJson, + "blockId" -> iEt.blockId.asJson, + "inclusionHeight" -> iEt.inclusionHeight.asJson, + "timestamp" -> iEt.timestamp.asJson, + "index" -> iEt.index.asJson, + "globalIndex" -> iEt.globalIndex.asJson, + "numConfirmations" -> iEt.numConfirmations.asJson, + "inputs" -> iEt.inputs.asJson, + "dataInputs" -> iEt.dataInputs.asJson, + "outputs" -> iEt.outputs.asJson, + "size" -> iEt.txSize.asJson + ) + } + + implicit val indexedTxSeqEncoder: Encoder[(Seq[IndexedErgoTransaction],Long)] = { iEtSeq => + Json.obj( + "items" -> iEtSeq._1.asJson, + "total" -> iEtSeq._2.asJson ) } @@ -506,6 +520,24 @@ trait ApiCodecs extends JsonCodecs { ) } + /*implicit val BalanceInfoEncoder: Encoder[BalanceInfo] = { bal => + Json.obj( + "confirmed" -> Json.obj( + "nanoErgs" -> bal.nanoErgs.asJson, + "tokens" -> bal.tokens.map(t => { + val iT: IndexedToken = history.typedModifierById[IndexedToken](bytesToId(t._1)).get + Json.obj( + "tokenId" -> iT.tokenId.asJson, + "amount" -> t._2.asJson, + "decimals" -> iT.decimals.asJson, + "name" -> iT.name.asJson + ) + }) + ), + "unconfirmed" -> Json.obj() // TODO + ) + }*/ + } trait ApiEncoderOption diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index afd1dfbf4e..d8771e222b 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -49,13 +49,13 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getHistory: Future[ErgoHistoryReader] = (readersHolder ? GetDataFromHistory[ErgoHistoryReader](r => r)).mapTo[ErgoHistoryReader] - private def getAddress(tree: ErgoTree)(implicit history: ErgoHistoryReader): Option[IndexedErgoAddress] = { + private def getAddress(tree: ErgoTree)(history: ErgoHistoryReader): Option[IndexedErgoAddress] = { history.typedModifierById[IndexedErgoAddress](bytesToId(IndexedErgoAddressSerializer.hashErgoTree(tree))) } - private def getAddress(addr: ErgoAddress)(implicit history: ErgoHistoryReader): Option[IndexedErgoAddress] = getAddress(addr.script) + private def getAddress(addr: ErgoAddress)(history: ErgoHistoryReader): Option[IndexedErgoAddress] = getAddress(addr.script)(history) - private def getTxById(id: ModifierId)(implicit history: ErgoHistoryReader): Option[IndexedErgoTransaction] = + private def getTxById(id: ModifierId)(history: ErgoHistoryReader): Option[IndexedErgoTransaction] = history.typedModifierById[IndexedErgoTransaction](id) match { case Some(tx) => Some(tx.retrieveBody(history)) case None => None @@ -74,7 +74,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting getTxById(history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(index))).get.m)(history) private def getLastTx(history: ErgoHistoryReader): IndexedErgoTransaction = - getTxByIndex(ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalTxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong)(history).get + getTxByIndex(ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalTxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong - 1)(history).get private def getTxByIndexF(index: Long): Future[Option[IndexedErgoTransaction]] = getHistory.map { history => @@ -85,11 +85,11 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting ApiResponse(getTxByIndexF(index)) } - private def getTxsByAddress(addr: ErgoAddress, offset: Int, limit: Int): Future[Seq[IndexedErgoTransaction]] = + private def getTxsByAddress(addr: ErgoAddress, offset: Int, limit: Int): Future[(Seq[IndexedErgoTransaction],Long)] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => addr.retrieveTxs(history, offset, limit) - case None => Seq.empty[IndexedErgoTransaction] + case Some(addr) => (addr.retrieveTxs(history, offset, limit).reverse, addr.txCount()) + case None => (Seq.empty[IndexedErgoTransaction], 0L) } } @@ -108,7 +108,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting cfor(0)(_ < limit, _ + 1) { i => txIds(i) = history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(base - limit + i))).get.m } - txIds + txIds.reverse } private def getTxRangeR: Route = (pathPrefix("transaction" / "range") & paging) { (offset, limit) => @@ -119,7 +119,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxById(id: ModifierId)(implicit history: ErgoHistoryReader): Option[IndexedErgoBox] = + private def getBoxById(id: ModifierId)(history: ErgoHistoryReader): Option[IndexedErgoBox] = history.typedModifierById[IndexedErgoBox](id) private def getBoxByIdF(id: ModifierId): Future[Option[IndexedErgoBox]] = @@ -143,11 +143,11 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting ApiResponse(getBoxByIndexF(index)) } - private def getBoxesByAddress(addr: ErgoAddress, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = + private def getBoxesByAddress(addr: ErgoAddress, offset: Int, limit: Int): Future[(Seq[IndexedErgoBox],Long)] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => addr.retrieveBoxes(history, offset, limit) - case None => Seq.empty[IndexedErgoBox] + case Some(addr) => (addr.retrieveBoxes(history, offset, limit).reverse, addr.boxCount()) + case None => (Seq.empty[IndexedErgoBox], 0L) } } @@ -162,7 +162,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => addr.retrieveUtxos(history, offset, limit) + case Some(addr) => addr.retrieveUtxos(history, offset, limit).reverse case None => Seq.empty[IndexedErgoBox] } } @@ -176,7 +176,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } private def getLastBox(history: ErgoHistoryReader): IndexedErgoBox = - getBoxByIndex(ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalBoxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong)(history).get + getBoxByIndex(ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalBoxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong - 1)(history).get private def getBoxRange(offset: Int, limit: Int): Future[Seq[ModifierId]] = getHistory.map { history => @@ -185,7 +185,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting cfor(0)(_ < limit, _ + 1) { i => boxIds(i) = history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(base - limit + i))).get.m } - boxIds + boxIds.reverse } private def getBoxRangeR: Route = (pathPrefix("box" / "range") & paging) { (offset, limit) => @@ -199,7 +199,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxesByErgoTree(tree: ErgoTree, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(tree)(history) match { - case Some(iEa) => iEa.retrieveBoxes(history, offset, limit) + case Some(iEa) => iEa.retrieveBoxes(history, offset, limit).reverse case None => Seq.empty[IndexedErgoBox] } } @@ -215,7 +215,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getBoxesByErgoTreeUnspent(tree: ErgoTree, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = getHistory.map { history => getAddress(tree)(history) match { - case Some(iEa) => iEa.retrieveUtxos(history, offset, limit) + case Some(iEa) => iEa.retrieveUtxos(history, offset, limit).reverse case None => Seq.empty[IndexedErgoBox] } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala new file mode 100644 index 0000000000..493d9ed718 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -0,0 +1,47 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.ErgoBox +import org.ergoplatform.ErgoBox.TokenId +import scorex.core.serialization.ScorexSerializer +import scorex.crypto.hash.Digest32 +import scorex.util.serialization.{Reader, Writer} +import spire.implicits.cfor + +import java.nio.ByteBuffer +import scala.collection.mutable + +class BalanceInfo(var nanoErgs: Long, + val tokens: mutable.HashMap[TokenId,Long]) + +object BalanceInfoSerializer extends ScorexSerializer[BalanceInfo] { + + override def serialize(bI: BalanceInfo, w: Writer): Unit = { + w.putLong(bI.nanoErgs) + w.putInt(bI.tokens.size) + val b: ByteBuffer = ByteBuffer.allocate(8) + val arr: Array[(TokenId,Long)] = bI.tokens.toArray + cfor(0)(_ < arr.length, _ + 1) { i => + w.putBytes(arr(i)._1 ++ b.putLong(arr(i)._2).array) + b.clear() + } + } + + override def parse(r: Reader): BalanceInfo = { + val nanoErgs: Long = r.getLong() + val tokensLen: Int = r.getInt() + val tokens: mutable.HashMap[TokenId,Long] = mutable.HashMap.empty[TokenId,Long] + val b: ByteBuffer = ByteBuffer.allocate(8) + cfor(0)(_ < tokensLen, _ + 1) { _ => + tokens.put(Digest32 @@ r.getBytes(32), b.put(r.getBytes(8)).getLong) + b.clear() + } + new BalanceInfo(nanoErgs, tokens) + } + +} + +object BalanceInfo { + def fromBox(box: ErgoBox): BalanceInfo = { + new BalanceInfo(box.value, mutable.HashMap.empty[TokenId,Long] ++= box.additionalTokens.toMap) + } +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index c5d1689044..094ce8ad38 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -13,7 +13,6 @@ import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessage import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.segmentTreshold import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.settings.{Algos, CacheSettings, ChainSettings} -import scorex.db.ByteArrayWrapper import scorex.util.{ModifierId, ScorexLogging, bytesToId} import java.nio.ByteBuffer @@ -124,18 +123,30 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) if(height != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => val inputId: ModifierId = bytesToId(tx.inputs(i).boxId) + var boxIndex: Int = 0 findBoxOpt(inputId) match { case Some(x) => // box found in last saveLimit modifiers, update boxes(x).asSpent(tx.id, height) + boxIndex = x tokens ++= boxes(x).box.additionalTokens.toArray case None => // box not found in last saveLimit modifiers history.typedModifierById[IndexedErgoBox](inputId) match { case Some(x) => // box found in DB, update boxes += x.asSpent(tx.id, height) + boxIndex = boxes.length - 1 tokens ++= x.box.additionalTokens.toArray case None => log.warn(s"Unknown box used as input: $inputId") // box not found at all (this shouldn't happen) } } + val treeHash: ModifierId = bytesToId(IndexedErgoAddressSerializer.hashErgoTree(boxes(boxIndex).box.ergoTree)) + findTreeOpt(treeHash) match { + case Some(x) => trees(x).addTx(globalTxIndex).spendBox(boxes(boxIndex).box) // address found in last saveLimit modifiers, update + case None => // address not found in last saveLimit modifiers + history.typedModifierById[IndexedErgoAddress](treeHash) match { + case Some(x) => trees += x.addTx(globalTxIndex).spendBox(boxes(boxIndex).box) // address found in DB, update + case None => // address not found at all (this shouldn't happen) + } + } } } @@ -148,11 +159,11 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // box by address val treeHash: ModifierId = bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)) findTreeOpt(treeHash) match { - case Some(x) => trees(x).addTx(globalTxIndex).addBox(globalBoxIndex) // address found in last saveLimit modifiers, update + case Some(x) => trees(x).addTx(globalTxIndex).addBox(boxes.last) // address found in last saveLimit modifiers, update case None => // address not found in last saveLimit modifiers history.typedModifierById[IndexedErgoAddress](treeHash) match { - case Some(x) => trees += x.addTx(globalTxIndex).addBox(globalBoxIndex) // address found in DB, update - case None => trees += IndexedErgoAddress(treeHash, ListBuffer(globalTxIndex), ListBuffer(globalBoxIndex)) // address not found at all, record + case Some(x) => trees += x.addTx(globalTxIndex).addBox(boxes.last) // address found in DB, update + case None => trees += IndexedErgoAddress(treeHash, ListBuffer(globalTxIndex), ListBuffer.empty[Long], Some(BalanceInfo.fromBox(box))).addBox(boxes.last) // address not found at all, record } } @@ -253,9 +264,9 @@ object ExtraIndexerRef { x } - val IndexedHeightKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("indexed height")).data - val GlobalTxIndexKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("txns height")).data - val GlobalBoxIndexKey: Array[Byte] = ByteArrayWrapper.apply(Algos.hash("boxes height")).data + val IndexedHeightKey: Array[Byte] = Algos.hash("indexed height") + val GlobalTxIndexKey: Array[Byte] = Algos.hash("txns height") + val GlobalBoxIndexKey: Array[Byte] = Algos.hash("boxes height") def apply(chainSettings: ChainSettings, cacheSettings: CacheSettings)(implicit system: ActorSystem): ActorRef = { val actor = system.actorOf(Props.create(classOf[ExtraIndex], chainSettings, cacheSettings)) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 2d7b280cf1..84913e4ac7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -1,10 +1,11 @@ package org.ergoplatform.nodeView.history.extra -import org.ergoplatform.ErgoAddress +import org.ergoplatform.ErgoBox.TokenId +import org.ergoplatform.{ErgoAddress, ErgoBox} import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes -import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getSegmentsForRange, segmentTreshold, slice} +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getBoxes, getSegmentsForRange, getTxs, segmentTreshold, slice} import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, txSegmentId} import org.ergoplatform.settings.Algos import scorex.core.ModifierTypeId @@ -18,7 +19,8 @@ import spire.syntax.all.cfor case class IndexedErgoAddress(treeHash: ModifierId, txs: ListBuffer[Long], - boxes: ListBuffer[Long]) extends BlockSection with ScorexLogging { + boxes: ListBuffer[Long], + balanceInfo: Option[BalanceInfo]) extends BlockSection with ScorexLogging { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = fastIdToBytes(treeHash) @@ -34,39 +36,37 @@ case class IndexedErgoAddress(treeHash: ModifierId, def boxCount(): Long = segmentTreshold * boxSegmentCount + boxes.length def retrieveTxs(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoTransaction] = { - if (offset + limit > txs.length && txSegmentCount > 0) { + if(offset + limit > txs.length && txSegmentCount > 0) { val range: Array[Int] = getSegmentsForRange(offset, limit) + val data: ListBuffer[Long] = ListBuffer.empty[Long] cfor(0)(_ < range.length, _ + 1) { i => - txs ++=: (history.typedModifierById[IndexedErgoAddress](txSegmentId(treeHash, txSegmentCount - range(i))) match { - case Some(iEa) => iEa.txs - case None => ListBuffer.empty[Long] - }) + history.typedModifierById[IndexedErgoAddress](txSegmentId(treeHash, txSegmentCount - range(i))).get.txs ++=: data } - } - slice(txs, offset, limit).map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray + getTxs(slice(data ++= (if(offset < txs.length) txs else Nil), offset % segmentTreshold, limit))(history) + } else + getTxs(slice(txs, offset, limit))(history) } def retrieveBoxes(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoBox] = { if(offset + limit > boxes.length && boxSegmentCount > 0) { val range: Array[Int] = getSegmentsForRange(offset, limit) + val data: ListBuffer[Long] = ListBuffer.empty[Long] cfor(0)(_ < range.length, _ + 1) { i => - boxes ++=: (history.typedModifierById[IndexedErgoAddress](boxSegmentId(treeHash, boxSegmentCount - range(i))) match { - case Some(iEb) => iEb.boxes - case None => ListBuffer.empty[Long] - }) + history.typedModifierById[IndexedErgoAddress](boxSegmentId(treeHash, boxSegmentCount - range(i))).get.boxes ++=: data } - } - slice(boxes, offset, limit).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray + getBoxes(slice(data ++= (if(offset < boxes.length) boxes else Nil), offset % segmentTreshold, limit))(history) + } else + getBoxes(slice(boxes, offset, limit))(history) } def retrieveUtxos(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoBox] = { val data: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] data ++= boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) - var segment = boxSegmentCount + var segment: Int = boxSegmentCount while(data.length < limit && segment > 0) { segment -= 1 - data ++=: history.typedModifierById[IndexedErgoAddress](boxSegmentId(treeHash, segment)).get.boxes - .map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) + history.typedModifierById[IndexedErgoAddress](boxSegmentId(treeHash, segment)).get.boxes + .map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) ++=: data } slice(data, offset, limit).toArray } @@ -76,8 +76,22 @@ case class IndexedErgoAddress(treeHash: ModifierId, this } - def addBox(box: Long): IndexedErgoAddress = { - boxes += box + def addBox(iEb: IndexedErgoBox): IndexedErgoAddress = { + boxes += iEb.globalIndex + balanceInfo.get.nanoErgs += iEb.box.value + cfor(0)(_ < iEb.box.additionalTokens.length, _ + 1) { i => + val id: TokenId = iEb.box.additionalTokens(i)._1 + balanceInfo.get.tokens.put(id, balanceInfo.get.tokens.getOrElse(id, 0L) + iEb.box.additionalTokens(i)._2) + } + this + } + + def spendBox(box: ErgoBox): IndexedErgoAddress = { + balanceInfo.get.nanoErgs -= box.value + cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => + val id: TokenId = box.additionalTokens(i)._1 + balanceInfo.get.tokens.put(id, balanceInfo.get.tokens.getOrElse(id, 0L) - box.additionalTokens(i)._2) + } this } @@ -85,12 +99,12 @@ case class IndexedErgoAddress(treeHash: ModifierId, require(segmentTreshold < txs.length || segmentTreshold < boxes.length, "address does not have enough transactions or boxes for segmentation") val data: ListBuffer[IndexedErgoAddress] = ListBuffer.empty[IndexedErgoAddress] if(segmentTreshold < txs.length) { - data += new IndexedErgoAddress(txSegmentId(treeHash, txSegmentCount), txs.take(segmentTreshold), ListBuffer.empty[Long]) + data += new IndexedErgoAddress(txSegmentId(treeHash, txSegmentCount), txs.take(segmentTreshold), ListBuffer.empty[Long], None) txSegmentCount += 1 txs.remove(0, segmentTreshold) } if(segmentTreshold < boxes.length) { - data += new IndexedErgoAddress(boxSegmentId(treeHash, boxSegmentCount), ListBuffer.empty[Long], boxes.take(segmentTreshold)) + data += new IndexedErgoAddress(boxSegmentId(treeHash, boxSegmentCount), ListBuffer.empty[Long], boxes.take(segmentTreshold), None) boxSegmentCount += 1 boxes.remove(0, segmentTreshold) } @@ -112,6 +126,7 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] cfor(0)(_ < iEa.txs.length, _ + 1) { i => w.putLong(iEa.txs(i))} w.putUInt(iEa.boxes.length) cfor(0)(_ < iEa.boxes.length, _ + 1) { i => w.putLong(iEa.boxes(i))} + w.putOption[BalanceInfo](iEa.balanceInfo)((ww, bI) => BalanceInfoSerializer.serialize(bI, ww)) w.putInt(iEa.boxSegmentCount) w.putInt(iEa.txSegmentCount) } @@ -124,7 +139,8 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] val boxesLen: Long = r.getUInt() val boxes: ListBuffer[Long] = ListBuffer.empty[Long] cfor(0)(_ < boxesLen, _ + 1) { _ => boxes += r.getLong()} - val iEa: IndexedErgoAddress = new IndexedErgoAddress(addressHash, txns, boxes) + val balanceInfo: Option[BalanceInfo] = r.getOption[BalanceInfo](BalanceInfoSerializer.parse(r)) + val iEa: IndexedErgoAddress = new IndexedErgoAddress(addressHash, txns, boxes, balanceInfo) iEa.boxSegmentCount = r.getInt() iEa.txSegmentCount = r.getInt() iEa @@ -138,9 +154,14 @@ object IndexedErgoAddress { val segmentTreshold: Int = 512 def getSegmentsForRange(offset: Int, limit: Int): Array[Int] = - (math.ceil((offset + limit) * 1F / segmentTreshold).toInt to math.ceil((offset + 1F) / segmentTreshold).toInt by -1).toArray.reverse + (math.max(math.ceil(offset * 1F / segmentTreshold).toInt, 1) to math.ceil((offset + limit) * 1F / segmentTreshold).toInt).toArray def slice[T](arr: Iterable[T], offset: Int, limit: Int): Iterable[T] = arr.slice(arr.size - offset - limit, arr.size - offset) + def getTxs(arr: Iterable[Long])(history: ErgoHistoryReader): Array[IndexedErgoTransaction] = + arr.map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray + + def getBoxes(arr: Iterable[Long])(history: ErgoHistoryReader): Array[IndexedErgoBox] = + arr.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index 5fc9c911a1..6b99bcd10d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -9,11 +9,11 @@ import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import scorex.core.serialization.ScorexSerializer import scorex.core.ModifierTypeId import scorex.util.serialization.{Reader, Writer} -import scorex.util.{ModifierId, ScorexLogging, bytesToId} +import scorex.util.{ModifierId, bytesToId} case class IndexedErgoTransaction(txid: ModifierId, height: Int, - globalIndex: Long) extends BlockSection with ScorexLogging { + globalIndex: Long) extends BlockSection { override val modifierTypeId: ModifierTypeId = IndexedErgoTransaction.modifierTypeId override def serializedId: Array[Byte] = fastIdToBytes(txid) From 0135bee2167f58c38c28b1382cfc0642216ae049 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 11 Sep 2022 04:29:35 +0200 Subject: [PATCH 23/90] bugfixes in balance tracking, added api endpoints --- src/main/resources/api/openapi.yaml | 108 +++++++++++++++--- .../org/ergoplatform/http/api/ApiCodecs.scala | 35 +++--- .../http/api/BlockchainApiRoute.scala | 40 ++++++- .../nodeView/history/extra/BalanceInfo.scala | 61 ++++++++-- .../nodeView/history/extra/ExtraIndexer.scala | 78 ++++++------- .../history/extra/IndexedErgoAddress.scala | 17 +-- .../nodeView/history/extra/IndexedToken.scala | 18 +-- 7 files changed, 251 insertions(+), 106 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 1e22ccf94b..1ae73d7884 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -200,6 +200,37 @@ components: example: 1 default: 1 + BalanceInfo: + type: object + description: Balance information + required: + - nanoErgs + - tokens + properties: + nanoErgs: + type: integer + format: int64 + description: Balance of nanoERGs + tokens: + type: array + description: Balance of tokens + items: + type: object + properties: + tokenId: + $ref: '#/components/schemas/ModifierId' + description: Identifier of the token + amount: + type: integer + format: int64 + description: Amount of the token + decimals: + type: integer + description: Number of decimals of the token + name: + type: string + description: Name of the token, if any + IndexedErgoBox: type: object description: Box indexed with extra information @@ -5328,10 +5359,16 @@ paths: content: application/json: schema: - type: array - description: Array of transactions - items: - $ref: '#/components/schemas/IndexedErgoTransaction' + type: object + properties: + items: + type: array + description: Array of transactions + items: + $ref: '#/components/schemas/IndexedErgoTransaction' + total: + type: integer + description: Total count of transactions '404': description: No transactions found for wanted address content: @@ -5486,10 +5523,16 @@ paths: content: application/json: schema: - type: array - description: Array of boxes - items: - $ref: '#/components/schemas/IndexedErgoTransaction' + type: object + properties: + items: + type: array + description: Array of boxes + items: + $ref: '#/components/schemas/IndexedErgoTransaction' + total: + type: integer + description: Total number of boxes '404': description: No boxes found for wanted address content: @@ -5631,13 +5674,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/IndexedErgoBox' - '404': - description: No box found with wanted ergotree - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' + type: object + properties: + items: + type: array + description: Array of boxes + items: + $ref: '#/components/schemas/IndexedErgoBox' + total: + type: integer + description: Total number of boxes default: description: Error content: @@ -5721,6 +5767,38 @@ paths: application/json: schema: $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + /blockchain/balance/{address}: + get: + summary: Retrieve confirmed and unconfirmed balance of an address + operationId: getAddressBalanceTotal + tags: + - blockchain + parameters: + - in: path + name: address + required: true + description: adderess with balance + schema: + $ref: '#/components/schemas/ErgoAddress' + responses: + '200': + description: balance information + content: + application/json: + schema: + type: object + properties: + confirmed: + $ref: '#/components/schemas/BalanceInfo' + unconfirmed: + $ref: '#/components/schemas/BalanceInfo' default: description: Error content: diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index 280b626e1b..3e17f44b4f 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -28,7 +28,7 @@ import sigmastate.interpreter._ import sigmastate.interpreter.CryptoConstants.EcPointType import io.circe.syntax._ import org.ergoplatform.http.api.requests.{CryptoResult, ExecuteRequest, HintExtractionRequest} -import org.ergoplatform.nodeView.history.extra.{ExtraIndexerRef, IndexedErgoBox, IndexedErgoTransaction, IndexedToken} +import org.ergoplatform.nodeView.history.extra.{BalanceInfo, ExtraIndexerRef, IndexedErgoBox, IndexedErgoTransaction, IndexedToken} import org.ergoplatform.wallet.interface4j.SecretString import scorex.crypto.authds.{LeafData, Side} import scorex.crypto.authds.merkle.MerkleProof @@ -520,23 +520,26 @@ trait ApiCodecs extends JsonCodecs { ) } - /*implicit val BalanceInfoEncoder: Encoder[BalanceInfo] = { bal => + implicit val BalanceInfoEncoder: Encoder[BalanceInfo] = { bal => Json.obj( - "confirmed" -> Json.obj( - "nanoErgs" -> bal.nanoErgs.asJson, - "tokens" -> bal.tokens.map(t => { - val iT: IndexedToken = history.typedModifierById[IndexedToken](bytesToId(t._1)).get - Json.obj( - "tokenId" -> iT.tokenId.asJson, - "amount" -> t._2.asJson, - "decimals" -> iT.decimals.asJson, - "name" -> iT.name.asJson - ) - }) - ), - "unconfirmed" -> Json.obj() // TODO + "nanoErgs" -> bal.nanoErgs.asJson, + "tokens" -> bal.tokens.map(token => { + Json.obj( + "tokenId" -> token._1.asJson, + "amount" -> token._2.asJson, + "decimals" -> bal.additionalTokenInfo(token._1)._2.asJson, + "name" -> bal.additionalTokenInfo(token._1)._1.asJson + ) + }).asJson ) - }*/ + } + + implicit val TotalBalanceInfoEncoder: Encoder[(BalanceInfo,BalanceInfo)] = { tBal => + Json.obj( + "confirmed" -> tBal._1.asJson, + "unconfirmed" -> tBal._2.asJson + ) + } } diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index d8771e222b..e6c8a3a501 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -4,10 +4,11 @@ import akka.actor.{ActorRef, ActorRefFactory} import akka.http.scaladsl.server.{Directive, Route} import akka.pattern.ask import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} -import org.ergoplatform.nodeView.ErgoReadersHolder.GetDataFromHistory +import org.ergoplatform.nodeView.ErgoReadersHolder.{GetDataFromHistory, GetReaders, Readers} import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{GlobalBoxIndexKey, GlobalTxIndexKey} import org.ergoplatform.nodeView.history.extra._ +import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader import org.ergoplatform.settings.ErgoSettings import scorex.core.api.http.ApiError.BadRequest import scorex.core.api.http.ApiResponse @@ -43,12 +44,16 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting getBoxRangeR ~ getBoxesByErgoTreeR ~ getBoxesByErgoTreeUnspentR ~ - getTokenInfoByIdR + getTokenInfoByIdR ~ + getAddressBalanceTotalR } private def getHistory: Future[ErgoHistoryReader] = (readersHolder ? GetDataFromHistory[ErgoHistoryReader](r => r)).mapTo[ErgoHistoryReader] + private def getHistoryWithMempool: Future[(ErgoHistoryReader,ErgoMemPoolReader)] = + (readersHolder ? GetReaders).mapTo[Readers].map(r => (r.h, r.m)) + private def getAddress(tree: ErgoTree)(history: ErgoHistoryReader): Option[IndexedErgoAddress] = { history.typedModifierById[IndexedErgoAddress](bytesToId(IndexedErgoAddressSerializer.hashErgoTree(tree))) } @@ -196,11 +201,11 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByErgoTree(tree: ErgoTree, offset: Int, limit: Int): Future[Seq[IndexedErgoBox]] = + private def getBoxesByErgoTree(tree: ErgoTree, offset: Int, limit: Int): Future[(Seq[IndexedErgoBox],Long)] = getHistory.map { history => getAddress(tree)(history) match { - case Some(iEa) => iEa.retrieveBoxes(history, offset, limit).reverse - case None => Seq.empty[IndexedErgoBox] + case Some(iEa) => (iEa.retrieveBoxes(history, offset, limit).reverse, iEa.boxCount()) + case None => (Seq.empty[IndexedErgoBox], 0L) } } @@ -238,4 +243,29 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting ApiResponse(getTokenInfoById(id)) } + private def getUnconfirmedForAddress(address: ErgoAddress)(mempool: ErgoMemPoolReader): BalanceInfo = { + val bal: BalanceInfo = BalanceInfo.empty + mempool.getAll.map(_.transaction).foreach(tx => { + tx.outputs.foreach(box => { + if(IndexedErgoBoxSerializer.getAddress(box.ergoTree).equals(address)) bal.add(box) + }) + }) + bal + } + + private def getAddressBalanceTotal(address: ErgoAddress): Future[(BalanceInfo,BalanceInfo)] = { + getHistoryWithMempool.map { case (history, mempool) => + getAddress(address)(history) match { + case Some(addr) => + (addr.balanceInfo.get.retreiveAdditionalTokenInfo(history), getUnconfirmedForAddress(address)(mempool).retreiveAdditionalTokenInfo(history)) + case None => + (BalanceInfo.empty, getUnconfirmedForAddress(address)(mempool).retreiveAdditionalTokenInfo(history)) + } + } + } + + private def getAddressBalanceTotalR: Route = (get & pathPrefix("balance") & ergoAddress) { address => + ApiResponse(getAddressBalanceTotal(address)) + } + } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala index 493d9ed718..dc00537e90 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -2,46 +2,85 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.TokenId +import org.ergoplatform.nodeView.history.ErgoHistoryReader import scorex.core.serialization.ScorexSerializer import scorex.crypto.hash.Digest32 +import scorex.util.bytesToId import scorex.util.serialization.{Reader, Writer} import spire.implicits.cfor -import java.nio.ByteBuffer import scala.collection.mutable +import scala.collection.JavaConverters.mapAsScalaMap class BalanceInfo(var nanoErgs: Long, - val tokens: mutable.HashMap[TokenId,Long]) + val tokens: mutable.Map[TokenId,Long]) { + + val additionalTokenInfo: mutable.Map[TokenId,(String,Int)] = mutable.Map.empty[TokenId,(String,Int)] + + def retreiveAdditionalTokenInfo(history: ErgoHistoryReader): BalanceInfo = { + additionalTokenInfo ++= tokens.map(token => { + val iT: IndexedToken = history.typedModifierById[IndexedToken](IndexedTokenSerializer.uniqueId(bytesToId(token._1))).get + (token._1,(iT.name,iT.decimals)) + }) + this + } + + def add(box: ErgoBox): BalanceInfo = { + nanoErgs += box.value + cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => + tokens.put(box.additionalTokens(i)._1, tokens.getOrElse(box.additionalTokens(i)._1, 0L) + box.additionalTokens(i)._2) + } + this + } + + def subtract(box: ErgoBox): BalanceInfo = { + nanoErgs -= box.value + cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => + val newVal: Long = tokens.getOrElse(box.additionalTokens(i)._1, 0L) - box.additionalTokens(i)._2 + if(newVal == 0) + tokens.remove(box.additionalTokens(i)._1) + else + tokens.put(box.additionalTokens(i)._1, newVal) + } + this + } + +} object BalanceInfoSerializer extends ScorexSerializer[BalanceInfo] { override def serialize(bI: BalanceInfo, w: Writer): Unit = { w.putLong(bI.nanoErgs) w.putInt(bI.tokens.size) - val b: ByteBuffer = ByteBuffer.allocate(8) val arr: Array[(TokenId,Long)] = bI.tokens.toArray cfor(0)(_ < arr.length, _ + 1) { i => - w.putBytes(arr(i)._1 ++ b.putLong(arr(i)._2).array) - b.clear() + w.putBytes(arr(i)._1) + w.putLong(arr(i)._2) } } override def parse(r: Reader): BalanceInfo = { val nanoErgs: Long = r.getLong() val tokensLen: Int = r.getInt() - val tokens: mutable.HashMap[TokenId,Long] = mutable.HashMap.empty[TokenId,Long] - val b: ByteBuffer = ByteBuffer.allocate(8) + val tokens: java.util.HashMap[TokenId,Long] = new java.util.HashMap[TokenId,Long] cfor(0)(_ < tokensLen, _ + 1) { _ => - tokens.put(Digest32 @@ r.getBytes(32), b.put(r.getBytes(8)).getLong) - b.clear() + tokens.put(Digest32 @@ r.getBytes(32), r.getLong()) } - new BalanceInfo(nanoErgs, tokens) + new BalanceInfo(nanoErgs, mapAsScalaMap(tokens)) } } object BalanceInfo { + def fromBox(box: ErgoBox): BalanceInfo = { - new BalanceInfo(box.value, mutable.HashMap.empty[TokenId,Long] ++= box.additionalTokens.toMap) + val tokens: java.util.HashMap[TokenId,Long] = new java.util.HashMap[TokenId,Long] + cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => + tokens.put(box.additionalTokens(i)._1, box.additionalTokens(i)._2) + } + new BalanceInfo(box.value, mapAsScalaMap(tokens)) } + + def empty: BalanceInfo = new BalanceInfo(0, mutable.Map.empty[TokenId,Long]) + } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 094ce8ad38..0b8a60acc7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -48,20 +48,50 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private val boxes: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] private val trees: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] + // input tokens in a tx private val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] - private def findBoxOpt(id: ModifierId): Option[Int] = { + // returns index of box in boxes + private def findAndSpendBox(id: ModifierId, txId: ModifierId, height: Int): Int = { cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(boxes(i).id == id) return Some(i) + if(boxes(i).id == id) { // box found in last saveLimit modifiers, update + tokens ++= boxes(i).asSpent(txId, height).box.additionalTokens.toArray + return i + } + } + history.typedModifierById[IndexedErgoBox](id) match { // box not found in last saveLimit modifiers + case Some(x) => // box found in DB, update + boxes += x.asSpent(txId, height) + tokens ++= x.box.additionalTokens.toArray + boxes.length - 1 + case None => // box not found at all (this shouldn't happen) + log.warn(s"Unknown box used as input: $id") + -1 } - None } - private def findTreeOpt(id: ModifierId): Option[Int] = { + private def findAndUpdateTree(id: ModifierId, boxToSpend: Option[ErgoBox]): Unit = { cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(trees(i).treeHash == id) return Some(i) + if(trees(i).treeHash == id) { // address found in last saveLimit modifiers + if(boxToSpend.isDefined) + trees(i).addTx(globalTxIndex).spendBox(boxToSpend.get) // spend box + else + trees(i).addTx(globalTxIndex).addBox(boxes.last) // receive box + return + } + } + history.typedModifierById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers + case Some(x) => + if(boxToSpend.isDefined) // address found in DB + trees += x.addTx(globalTxIndex).spendBox(boxToSpend.get) // spend box + else + trees += x.addTx(globalTxIndex).addBox(boxes.last) // receive box + case None => // address not found at all + if(boxToSpend.isEmpty) + trees += IndexedErgoAddress(id, ListBuffer(globalTxIndex), ListBuffer.empty[Long], Some(BalanceInfo.empty)).addBox(boxes.last) // receive box + else + log.warn(s"Unknown address spent box ${boxes.last.id}") // spend box should never happen by an unknown address } - None } private def modCount: Int = general.length + boxes.length + trees.length @@ -123,30 +153,8 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) if(height != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => val inputId: ModifierId = bytesToId(tx.inputs(i).boxId) - var boxIndex: Int = 0 - findBoxOpt(inputId) match { - case Some(x) => // box found in last saveLimit modifiers, update - boxes(x).asSpent(tx.id, height) - boxIndex = x - tokens ++= boxes(x).box.additionalTokens.toArray - case None => // box not found in last saveLimit modifiers - history.typedModifierById[IndexedErgoBox](inputId) match { - case Some(x) => // box found in DB, update - boxes += x.asSpent(tx.id, height) - boxIndex = boxes.length - 1 - tokens ++= x.box.additionalTokens.toArray - case None => log.warn(s"Unknown box used as input: $inputId") // box not found at all (this shouldn't happen) - } - } - val treeHash: ModifierId = bytesToId(IndexedErgoAddressSerializer.hashErgoTree(boxes(boxIndex).box.ergoTree)) - findTreeOpt(treeHash) match { - case Some(x) => trees(x).addTx(globalTxIndex).spendBox(boxes(boxIndex).box) // address found in last saveLimit modifiers, update - case None => // address not found in last saveLimit modifiers - history.typedModifierById[IndexedErgoAddress](treeHash) match { - case Some(x) => trees += x.addTx(globalTxIndex).spendBox(boxes(boxIndex).box) // address found in DB, update - case None => // address not found at all (this shouldn't happen) - } - } + val boxIndex: Int = findAndSpendBox(inputId, tx.id, height) + if(boxIndex >= 0) findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(boxes(boxIndex).box.ergoTree)), Some(boxes(boxIndex).box)) // spend box and add tx } } @@ -157,15 +165,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number // box by address - val treeHash: ModifierId = bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)) - findTreeOpt(treeHash) match { - case Some(x) => trees(x).addTx(globalTxIndex).addBox(boxes.last) // address found in last saveLimit modifiers, update - case None => // address not found in last saveLimit modifiers - history.typedModifierById[IndexedErgoAddress](treeHash) match { - case Some(x) => trees += x.addTx(globalTxIndex).addBox(boxes.last) // address found in DB, update - case None => trees += IndexedErgoAddress(treeHash, ListBuffer(globalTxIndex), ListBuffer.empty[Long], Some(BalanceInfo.fromBox(box))).addBox(boxes.last) // address not found at all, record - } - } + findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)), None) // check if box is creating a new token, if yes record it if(box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 84913e4ac7..e47f80c094 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -1,6 +1,5 @@ package org.ergoplatform.nodeView.history.extra -import org.ergoplatform.ErgoBox.TokenId import org.ergoplatform.{ErgoAddress, ErgoBox} import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader @@ -10,7 +9,7 @@ import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{box import org.ergoplatform.settings.Algos import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer -import scorex.util.{ModifierId, ScorexLogging, bytesToId} +import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.ErgoTree @@ -20,7 +19,7 @@ import spire.syntax.all.cfor case class IndexedErgoAddress(treeHash: ModifierId, txs: ListBuffer[Long], boxes: ListBuffer[Long], - balanceInfo: Option[BalanceInfo]) extends BlockSection with ScorexLogging { + balanceInfo: Option[BalanceInfo]) extends BlockSection { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = fastIdToBytes(treeHash) @@ -78,20 +77,12 @@ case class IndexedErgoAddress(treeHash: ModifierId, def addBox(iEb: IndexedErgoBox): IndexedErgoAddress = { boxes += iEb.globalIndex - balanceInfo.get.nanoErgs += iEb.box.value - cfor(0)(_ < iEb.box.additionalTokens.length, _ + 1) { i => - val id: TokenId = iEb.box.additionalTokens(i)._1 - balanceInfo.get.tokens.put(id, balanceInfo.get.tokens.getOrElse(id, 0L) + iEb.box.additionalTokens(i)._2) - } + balanceInfo.get.add(iEb.box) this } def spendBox(box: ErgoBox): IndexedErgoAddress = { - balanceInfo.get.nanoErgs -= box.value - cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => - val id: TokenId = box.additionalTokens(i)._1 - balanceInfo.get.tokens.put(id, balanceInfo.get.tokens.getOrElse(id, 0L) - box.additionalTokens(i)._2) - } + balanceInfo.get.subtract(box) this } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 3b7a340f54..1c727e5370 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -67,16 +67,18 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { box.additionalTokens(0)._2, new String(box.additionalRegisters(R4).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), new String(box.additionalRegisters(R5).asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8"), - math.max(getDecimals(box.additionalRegisters(R6)), 0)) + getDecimals(box.additionalRegisters(R6))) override def serialize(iT: IndexedToken, w: Writer): Unit = { w.putBytes(fastIdToBytes(iT.tokenId)) w.putBytes(fastIdToBytes(iT.boxId)) w.putULong(iT.amount) - w.putUShort(iT.name.length) - w.putBytes(iT.name.getBytes("UTF-8")) - w.putUShort(iT.description.length) - w.putBytes(iT.description.getBytes("UTF-8")) + val name: Array[Byte] = iT.name.getBytes("UTF-8") + w.putUShort(name.length) + w.putBytes(name) + val description: Array[Byte] = iT.description.getBytes("UTF-8") + w.putUShort(description.length) + w.putBytes(description) w.putInt(iT.decimals) } @@ -84,8 +86,10 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { val tokenId: ModifierId = bytesToId(r.getBytes(32)) val boxId: ModifierId = bytesToId(r.getBytes(32)) val amount: Long = r.getULong() - val name: String = new String(r.getBytes(r.getUShort), "UTF-8") - val description: String = new String(r.getBytes(r.getUShort), "UTF-8") + val nameLen: Int = r.getUShort() + val name: String = new String(r.getBytes(nameLen), "UTF-8") + val descLen: Int = r.getUShort() + val description: String = new String(r.getBytes(descLen), "UTF-8") val decimals: Int = r.getInt() IndexedToken(tokenId, boxId, amount, name, description, decimals) } From 3f7ff3d1c9c5c334e94a689cb283c9832b342b99 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 11 Sep 2022 16:07:53 +0200 Subject: [PATCH 24/90] smaller optimizations --- .../nodeView/history/extra/BalanceInfo.scala | 8 -------- .../nodeView/history/extra/ExtraIndexer.scala | 13 ++++++------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala index dc00537e90..05cfeec33d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -73,14 +73,6 @@ object BalanceInfoSerializer extends ScorexSerializer[BalanceInfo] { object BalanceInfo { - def fromBox(box: ErgoBox): BalanceInfo = { - val tokens: java.util.HashMap[TokenId,Long] = new java.util.HashMap[TokenId,Long] - cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => - tokens.put(box.additionalTokens(i)._1, box.additionalTokens(i)._2) - } - new BalanceInfo(box.value, mapAsScalaMap(tokens)) - } - def empty: BalanceInfo = new BalanceInfo(0, mutable.Map.empty[TokenId,Long]) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 0b8a60acc7..cadbdaa399 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -1,7 +1,7 @@ package org.ergoplatform.nodeView.history.extra import akka.actor.{Actor, ActorRef, ActorSystem, Props} -import org.ergoplatform.ErgoBox.TokenId +import org.ergoplatform.ErgoBox.{BoxId, TokenId} import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} import org.ergoplatform.modifiers.history.BlockTransactions @@ -52,20 +52,20 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] // returns index of box in boxes - private def findAndSpendBox(id: ModifierId, txId: ModifierId, height: Int): Int = { + private def findAndSpendBox(id: BoxId, txId: ModifierId, height: Int): Int = { cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(boxes(i).id == id) { // box found in last saveLimit modifiers, update + if(java.util.Arrays.equals(boxes(i).serializedId, id)) { // box found in last saveLimit modifiers, update tokens ++= boxes(i).asSpent(txId, height).box.additionalTokens.toArray return i } } - history.typedModifierById[IndexedErgoBox](id) match { // box not found in last saveLimit modifiers + history.typedModifierById[IndexedErgoBox](bytesToId(id)) match { // box not found in last saveLimit modifiers case Some(x) => // box found in DB, update boxes += x.asSpent(txId, height) tokens ++= x.box.additionalTokens.toArray boxes.length - 1 case None => // box not found at all (this shouldn't happen) - log.warn(s"Unknown box used as input: $id") + log.warn(s"Unknown box used as input: ${bytesToId(id)}") -1 } } @@ -152,8 +152,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) //process transaction inputs if(height != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => - val inputId: ModifierId = bytesToId(tx.inputs(i).boxId) - val boxIndex: Int = findAndSpendBox(inputId, tx.id, height) + val boxIndex: Int = findAndSpendBox(tx.inputs(i).boxId, tx.id, height) if(boxIndex >= 0) findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(boxes(boxIndex).box.ergoTree)), Some(boxes(boxIndex).box)) // spend box and add tx } } From 40fe104320b255f0ff0bfdf3bee84abebc3d9cd9 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Mon, 12 Sep 2022 01:05:19 +0200 Subject: [PATCH 25/90] small refactor --- .../nodeView/history/extra/ExtraIndexer.scala | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index cadbdaa399..331cd50b3f 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -215,18 +215,15 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) log.info("Indexer caught up with chain") } - override def preStart(): Unit = { + override def preStart(): Unit = context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) - } - override def postStop(): Unit = { + override def postStop(): Unit = log.info(s"Stopped extra indexer at height $lastWroteToDB") - } override def receive: Receive = { - case SemanticallySuccessfulModifier(fb: ErgoFullBlock) => - if(caughtUp) // after the indexer caught up with the chain, stay up to date - index(fb.blockTransactions, fb.height) + case SemanticallySuccessfulModifier(fb: ErgoFullBlock) if caughtUp => + index(fb.blockTransactions, fb.height) // after the indexer caught up with the chain, stay up to date case Start(history: ErgoHistory) => _history = history run() From bf3c5a9871f9d7bf99f664c019622a5a761ae435 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Tue, 13 Sep 2022 03:05:05 +0200 Subject: [PATCH 26/90] [UNTESTED] bugfix in balance tracking --- .../http/api/BlockchainApiRoute.scala | 4 +-- .../nodeView/history/extra/BalanceInfo.scala | 35 +++++++------------ .../nodeView/history/extra/ExtraIndexer.scala | 2 +- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index e6c8a3a501..d4f84e3c2f 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -244,7 +244,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } private def getUnconfirmedForAddress(address: ErgoAddress)(mempool: ErgoMemPoolReader): BalanceInfo = { - val bal: BalanceInfo = BalanceInfo.empty + val bal: BalanceInfo = new BalanceInfo mempool.getAll.map(_.transaction).foreach(tx => { tx.outputs.foreach(box => { if(IndexedErgoBoxSerializer.getAddress(box.ergoTree).equals(address)) bal.add(box) @@ -259,7 +259,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting case Some(addr) => (addr.balanceInfo.get.retreiveAdditionalTokenInfo(history), getUnconfirmedForAddress(address)(mempool).retreiveAdditionalTokenInfo(history)) case None => - (BalanceInfo.empty, getUnconfirmedForAddress(address)(mempool).retreiveAdditionalTokenInfo(history)) + (new BalanceInfo, getUnconfirmedForAddress(address)(mempool).retreiveAdditionalTokenInfo(history)) } } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala index 05cfeec33d..01cfdc09e7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -10,12 +10,11 @@ import scorex.util.serialization.{Reader, Writer} import spire.implicits.cfor import scala.collection.mutable -import scala.collection.JavaConverters.mapAsScalaMap -class BalanceInfo(var nanoErgs: Long, - val tokens: mutable.Map[TokenId,Long]) { +class BalanceInfo(var nanoErgs: Long = 0L, + val tokens: mutable.HashMap[TokenId,Long] = mutable.HashMap.empty[TokenId,Long]) { - val additionalTokenInfo: mutable.Map[TokenId,(String,Int)] = mutable.Map.empty[TokenId,(String,Int)] + val additionalTokenInfo: mutable.HashMap[TokenId,(String,Int)] = mutable.HashMap.empty[TokenId,(String,Int)] def retreiveAdditionalTokenInfo(history: ErgoHistoryReader): BalanceInfo = { additionalTokenInfo ++= tokens.map(token => { @@ -25,24 +24,22 @@ class BalanceInfo(var nanoErgs: Long, this } - def add(box: ErgoBox): BalanceInfo = { + def add(box: ErgoBox): Unit = { nanoErgs += box.value cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => tokens.put(box.additionalTokens(i)._1, tokens.getOrElse(box.additionalTokens(i)._1, 0L) + box.additionalTokens(i)._2) } - this } - def subtract(box: ErgoBox): BalanceInfo = { + def subtract(box: ErgoBox): Unit = { nanoErgs -= box.value cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => - val newVal: Long = tokens.getOrElse(box.additionalTokens(i)._1, 0L) - box.additionalTokens(i)._2 + val newVal: Long = tokens(box.additionalTokens(i)._1) - box.additionalTokens(i)._2 if(newVal == 0) tokens.remove(box.additionalTokens(i)._1) else tokens.put(box.additionalTokens(i)._1, newVal) } - this } } @@ -52,27 +49,19 @@ object BalanceInfoSerializer extends ScorexSerializer[BalanceInfo] { override def serialize(bI: BalanceInfo, w: Writer): Unit = { w.putLong(bI.nanoErgs) w.putInt(bI.tokens.size) - val arr: Array[(TokenId,Long)] = bI.tokens.toArray - cfor(0)(_ < arr.length, _ + 1) { i => - w.putBytes(arr(i)._1) - w.putLong(arr(i)._2) + bI.tokens.foreach { case (id,amount) => + w.putBytes(id) + w.putLong(amount) } } override def parse(r: Reader): BalanceInfo = { - val nanoErgs: Long = r.getLong() + val bI: BalanceInfo = new BalanceInfo(r.getLong()) val tokensLen: Int = r.getInt() - val tokens: java.util.HashMap[TokenId,Long] = new java.util.HashMap[TokenId,Long] cfor(0)(_ < tokensLen, _ + 1) { _ => - tokens.put(Digest32 @@ r.getBytes(32), r.getLong()) + bI.tokens.put(Digest32 @@ r.getBytes(32), r.getLong()) } - new BalanceInfo(nanoErgs, mapAsScalaMap(tokens)) + bI } } - -object BalanceInfo { - - def empty: BalanceInfo = new BalanceInfo(0, mutable.Map.empty[TokenId,Long]) - -} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 331cd50b3f..66fbffd5e0 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -88,7 +88,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) trees += x.addTx(globalTxIndex).addBox(boxes.last) // receive box case None => // address not found at all if(boxToSpend.isEmpty) - trees += IndexedErgoAddress(id, ListBuffer(globalTxIndex), ListBuffer.empty[Long], Some(BalanceInfo.empty)).addBox(boxes.last) // receive box + trees += IndexedErgoAddress(id, ListBuffer(globalTxIndex), ListBuffer.empty[Long], Some(new BalanceInfo)).addBox(boxes.last) // receive box else log.warn(s"Unknown address spent box ${boxes.last.id}") // spend box should never happen by an unknown address } From af035ac0231bbf77bc797dbe9df3b9817c5dd5ab Mon Sep 17 00:00:00 2001 From: jellymlg Date: Thu, 15 Sep 2022 00:35:23 +0200 Subject: [PATCH 27/90] balance tracking mostly working and optimized --- .../nodeView/history/extra/BalanceInfo.scala | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala index 01cfdc09e7..56ff45b35d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -1,44 +1,58 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox -import org.ergoplatform.ErgoBox.TokenId import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import scorex.core.serialization.ScorexSerializer -import scorex.crypto.hash.Digest32 -import scorex.util.bytesToId +import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} import spire.implicits.cfor import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer class BalanceInfo(var nanoErgs: Long = 0L, - val tokens: mutable.HashMap[TokenId,Long] = mutable.HashMap.empty[TokenId,Long]) { + val tokens: ArrayBuffer[(ModifierId,Long)] = ArrayBuffer.empty[(ModifierId,Long)]) { - val additionalTokenInfo: mutable.HashMap[TokenId,(String,Int)] = mutable.HashMap.empty[TokenId,(String,Int)] + val additionalTokenInfo: mutable.HashMap[ModifierId,(String,Int)] = mutable.HashMap.empty[ModifierId,(String,Int)] def retreiveAdditionalTokenInfo(history: ErgoHistoryReader): BalanceInfo = { additionalTokenInfo ++= tokens.map(token => { - val iT: IndexedToken = history.typedModifierById[IndexedToken](IndexedTokenSerializer.uniqueId(bytesToId(token._1))).get + val iT: IndexedToken = history.typedModifierById[IndexedToken](IndexedTokenSerializer.uniqueId(token._1)).get (token._1,(iT.name,iT.decimals)) }) this } + private def index(id: ModifierId): Int = { + cfor(0)(_ < tokens.length, _ + 1) { i => + if(tokens(i)._1 == id) return i + } + -1 + } + def add(box: ErgoBox): Unit = { nanoErgs += box.value cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => - tokens.put(box.additionalTokens(i)._1, tokens.getOrElse(box.additionalTokens(i)._1, 0L) + box.additionalTokens(i)._2) + val id: ModifierId = bytesToId(box.additionalTokens(i)._1) + val n: Int = index(id) + if(n >= 0) + tokens(n) = Tuple2(id, tokens(n)._2 + box.additionalTokens(i)._2) + else + tokens += Tuple2(id, box.additionalTokens(i)._2) } } def subtract(box: ErgoBox): Unit = { nanoErgs -= box.value cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => - val newVal: Long = tokens(box.additionalTokens(i)._1) - box.additionalTokens(i)._2 + val id: ModifierId = bytesToId(box.additionalTokens(i)._1) + val n: Int = index(id) + val newVal: Long = tokens(n)._2 - box.additionalTokens(i)._2 if(newVal == 0) - tokens.remove(box.additionalTokens(i)._1) + tokens.remove(n) else - tokens.put(box.additionalTokens(i)._1, newVal) + tokens(n) = (id, newVal) } } @@ -48,10 +62,10 @@ object BalanceInfoSerializer extends ScorexSerializer[BalanceInfo] { override def serialize(bI: BalanceInfo, w: Writer): Unit = { w.putLong(bI.nanoErgs) - w.putInt(bI.tokens.size) - bI.tokens.foreach { case (id,amount) => - w.putBytes(id) - w.putLong(amount) + w.putInt(bI.tokens.length) + cfor(0)(_ < bI.tokens.length, _ + 1) { i => + w.putBytes(fastIdToBytes(bI.tokens(i)._1)) + w.putLong(bI.tokens(i)._2) } } @@ -59,7 +73,7 @@ object BalanceInfoSerializer extends ScorexSerializer[BalanceInfo] { val bI: BalanceInfo = new BalanceInfo(r.getLong()) val tokensLen: Int = r.getInt() cfor(0)(_ < tokensLen, _ + 1) { _ => - bI.tokens.put(Digest32 @@ r.getBytes(32), r.getLong()) + bI.tokens += Tuple2(bytesToId(r.getBytes(32)), r.getLong()) } bI } From ac19515fc4b932db0771fb7dce3b072dba7445e5 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sat, 17 Sep 2022 01:16:55 +0200 Subject: [PATCH 28/90] refactored indexing order to match explorer --- .../nodeView/history/extra/BalanceInfo.scala | 37 +++--- .../nodeView/history/extra/ExtraIndexer.scala | 114 +++++++++++------- 2 files changed, 90 insertions(+), 61 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala index 56ff45b35d..eeb624df38 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -4,7 +4,7 @@ import org.ergoplatform.ErgoBox import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import scorex.core.serialization.ScorexSerializer -import scorex.util.{ModifierId, bytesToId} +import scorex.util.{ModifierId, ScorexLogging, bytesToId} import scorex.util.serialization.{Reader, Writer} import spire.implicits.cfor @@ -12,7 +12,7 @@ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer class BalanceInfo(var nanoErgs: Long = 0L, - val tokens: ArrayBuffer[(ModifierId,Long)] = ArrayBuffer.empty[(ModifierId,Long)]) { + val tokens: ArrayBuffer[(ModifierId,Long)] = ArrayBuffer.empty[(ModifierId,Long)]) extends ScorexLogging { val additionalTokenInfo: mutable.HashMap[ModifierId,(String,Int)] = mutable.HashMap.empty[ModifierId,(String,Int)] @@ -24,35 +24,38 @@ class BalanceInfo(var nanoErgs: Long = 0L, this } - private def index(id: ModifierId): Int = { + private def index(id: ModifierId): Option[Int] = { cfor(0)(_ < tokens.length, _ + 1) { i => - if(tokens(i)._1 == id) return i + if(tokens(i)._1 == id) return Some(i) } - -1 + None } def add(box: ErgoBox): Unit = { nanoErgs += box.value cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => val id: ModifierId = bytesToId(box.additionalTokens(i)._1) - val n: Int = index(id) - if(n >= 0) - tokens(n) = Tuple2(id, tokens(n)._2 + box.additionalTokens(i)._2) - else - tokens += Tuple2(id, box.additionalTokens(i)._2) + index(id) match { + case Some(n) => tokens(n) = Tuple2(id, tokens(n)._2 + box.additionalTokens(i)._2) + case None => tokens += Tuple2(id, box.additionalTokens(i)._2) + } } } def subtract(box: ErgoBox): Unit = { - nanoErgs -= box.value + nanoErgs = math.max(nanoErgs - box.value, 0) cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => val id: ModifierId = bytesToId(box.additionalTokens(i)._1) - val n: Int = index(id) - val newVal: Long = tokens(n)._2 - box.additionalTokens(i)._2 - if(newVal == 0) - tokens.remove(n) - else - tokens(n) = (id, newVal) + index(id) match { + case Some(n) => + val newVal: Long = tokens(n)._2 - box.additionalTokens(i)._2 + if(newVal == 0) + tokens.remove(n) + else + tokens(n) = (id, newVal) + case None => log.warn(s"Failed to subtract token $id from address ${IndexedErgoBoxSerializer.getAddress(box.ergoTree)}") + } + } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 66fbffd5e0..a12e3eb2f3 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -51,46 +51,54 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // input tokens in a tx private val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] + private def findBox(id: BoxId): Option[Int] = { + cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first + if (java.util.Arrays.equals(boxes(i).serializedId, id)) + return Some(i) // box found in last saveLimit modifiers + } + None + } + // returns index of box in boxes private def findAndSpendBox(id: BoxId, txId: ModifierId, height: Int): Int = { - cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(java.util.Arrays.equals(boxes(i).serializedId, id)) { // box found in last saveLimit modifiers, update + findBox(id) match { + case Some(i) => tokens ++= boxes(i).asSpent(txId, height).box.additionalTokens.toArray - return i - } - } - history.typedModifierById[IndexedErgoBox](bytesToId(id)) match { // box not found in last saveLimit modifiers - case Some(x) => // box found in DB, update - boxes += x.asSpent(txId, height) - tokens ++= x.box.additionalTokens.toArray - boxes.length - 1 - case None => // box not found at all (this shouldn't happen) - log.warn(s"Unknown box used as input: ${bytesToId(id)}") - -1 + i + case None => + history.typedModifierById[IndexedErgoBox](bytesToId(id)) match { // box not found in last saveLimit modifiers + case Some(x) => // box found in DB, update + boxes += x.asSpent(txId, height) + tokens ++= x.box.additionalTokens.toArray + boxes.length - 1 + case None => // box not found at all (this shouldn't happen) + log.warn(s"Unknown box used as input: ${bytesToId(id)}") + -1 + } } } - private def findAndUpdateTree(id: ModifierId, boxToSpend: Option[ErgoBox]): Unit = { + private def findAndUpdateTree(id: ModifierId, txIndex: Long, spendOrReceive: Either[ErgoBox,IndexedErgoBox]): Unit = { cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first if(trees(i).treeHash == id) { // address found in last saveLimit modifiers - if(boxToSpend.isDefined) - trees(i).addTx(globalTxIndex).spendBox(boxToSpend.get) // spend box - else - trees(i).addTx(globalTxIndex).addBox(boxes.last) // receive box + spendOrReceive match { + case Left(box) => trees(i).addTx(txIndex).spendBox(box) // spend box + case Right(iEb) => trees(i).addTx(txIndex).addBox(iEb) // receive box + } return } } history.typedModifierById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers case Some(x) => - if(boxToSpend.isDefined) // address found in DB - trees += x.addTx(globalTxIndex).spendBox(boxToSpend.get) // spend box - else - trees += x.addTx(globalTxIndex).addBox(boxes.last) // receive box + spendOrReceive match { + case Left(box) => trees += x.addTx(txIndex).spendBox(box) // spend box + case Right(iEb) => trees += x.addTx(txIndex).addBox(iEb) // receive box + } case None => // address not found at all - if(boxToSpend.isEmpty) - trees += IndexedErgoAddress(id, ListBuffer(globalTxIndex), ListBuffer.empty[Long], Some(new BalanceInfo)).addBox(boxes.last) // receive box - else - log.warn(s"Unknown address spent box ${boxes.last.id}") // spend box should never happen by an unknown address + spendOrReceive match { + case Left(box) => log.warn(s"Unknown address spent box ${bytesToId(box.id)}") // spend box should never happen by an unknown address + case Right(iEb) => trees += IndexedErgoAddress(id, ListBuffer(txIndex), ListBuffer.empty[Long], Some(new BalanceInfo)).addBox(iEb) // receive box + } } } @@ -139,7 +147,8 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) var boxCount: Int = 0 - cfor(0)(_ < bt.txs.size, _ + 1) { n => + // record transactions and boxes + cfor(0)(_ < bt.txs.length, _ + 1) { n => val tx: ErgoTransaction = bt.txs(n) @@ -147,40 +156,55 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) general += IndexedErgoTransaction(tx.id, height, globalTxIndex) general += NumericTxIndex(globalTxIndex, tx.id) + //process transaction outputs + cfor(0)(_ < tx.outputs.length, _ + 1) { i => + + val box: ErgoBox = tx.outputs(i) + + boxes += new IndexedErgoBox(Some(height), None, None, box, globalBoxIndex) // box by id + general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number + + globalBoxIndex += 1 + boxCount += 1 + + } + + globalTxIndex += 1 + + } + + // process transactions and boxes by address backwards because the explorer does this + cfor(bt.txs.length - 1)(_ >= 0, _ - 1) { n => + + val tx: ErgoTransaction = bt.txs(n) + tokens.clear() - //process transaction inputs + //process transaction inputs for addresses and tokens if(height != 1) { //only after 1st block (skip genesis box) - cfor(0)(_ < tx.inputs.size, _ + 1) { i => + cfor(0)(_ < tx.inputs.length, _ + 1) { i => val boxIndex: Int = findAndSpendBox(tx.inputs(i).boxId, tx.id, height) - if(boxIndex >= 0) findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(boxes(boxIndex).box.ergoTree)), Some(boxes(boxIndex).box)) // spend box and add tx + if(boxIndex >= 0) findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(boxes(boxIndex).box.ergoTree)), globalTxIndex - n - 1, Left(boxes(boxIndex).box)) // spend box and add tx } } - //process transaction outputs - cfor(0)(_ < tx.outputs.size, _ + 1) { i => + //process transaction outputs for addresses and tokens + cfor(0)(_ < tx.outputs.length, _ + 1) { i => + val box: ErgoBox = tx.outputs(i) - boxes += new IndexedErgoBox(Some(height), None, None, box, globalBoxIndex) // box by id - general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number // box by address - findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)), None) + findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)), globalTxIndex - n - 1, Right(boxes(findBox(box.id).get))) // check if box is creating a new token, if yes record it if(box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => - if(!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) { + if(!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) general += IndexedTokenSerializer.fromBox(box) - } } - globalBoxIndex += 1 - boxCount += 1 - } - globalTxIndex += 1 - } log.info(s"Buffered block #$height / $chainHeight [txs: ${bt.txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") @@ -193,7 +217,9 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) history.fullBlockHeight == history.headersHeight) // write to db every block after caught up saveProgress() - }else if(modCount >= saveLimit) saveProgress() // active syncing, write to db after modifier limit + }else + if(modCount >= saveLimit) + saveProgress() // active syncing, write to db after modifier limit } @@ -219,7 +245,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) override def postStop(): Unit = - log.info(s"Stopped extra indexer at height $lastWroteToDB") + log.info(s"Stopped extra indexer at height ${if(lastWroteToDB > 0) lastWroteToDB else indexedHeight}") override def receive: Receive = { case SemanticallySuccessfulModifier(fb: ErgoFullBlock) if caughtUp => From 8789891960d7283ad46eabae76a369d2818c47e1 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sat, 17 Sep 2022 05:19:54 +0200 Subject: [PATCH 29/90] different approach for indexing order, big speed improvement --- .../nodeView/history/extra/ExtraIndexer.scala | 62 ++++++++----------- .../history/extra/IndexedErgoAddress.scala | 20 ++++-- 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index a12e3eb2f3..7736e21b52 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -78,12 +78,12 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } } - private def findAndUpdateTree(id: ModifierId, txIndex: Long, spendOrReceive: Either[ErgoBox,IndexedErgoBox]): Unit = { + private def findAndUpdateTree(id: ModifierId, spendOrReceive: Either[ErgoBox,IndexedErgoBox]): Unit = { cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first if(trees(i).treeHash == id) { // address found in last saveLimit modifiers spendOrReceive match { - case Left(box) => trees(i).addTx(txIndex).spendBox(box) // spend box - case Right(iEb) => trees(i).addTx(txIndex).addBox(iEb) // receive box + case Left(box) => trees(i).addTx(globalTxIndex).spendBox(box) // spend box + case Right(iEb) => trees(i).addTx(globalTxIndex).addBox(iEb) // receive box } return } @@ -91,13 +91,13 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) history.typedModifierById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers case Some(x) => spendOrReceive match { - case Left(box) => trees += x.addTx(txIndex).spendBox(box) // spend box - case Right(iEb) => trees += x.addTx(txIndex).addBox(iEb) // receive box + case Left(box) => trees += x.addTx(globalTxIndex).spendBox(box) // spend box + case Right(iEb) => trees += x.addTx(globalTxIndex).addBox(iEb) // receive box } case None => // address not found at all spendOrReceive match { case Left(box) => log.warn(s"Unknown address spent box ${bytesToId(box.id)}") // spend box should never happen by an unknown address - case Right(iEb) => trees += IndexedErgoAddress(id, ListBuffer(txIndex), ListBuffer.empty[Long], Some(new BalanceInfo)).addBox(iEb) // receive box + case Right(iEb) => trees += IndexedErgoAddress(id, ListBuffer(globalTxIndex), ListBuffer.empty[Long], Some(new BalanceInfo)).addBox(iEb) // receive box } } } @@ -111,7 +111,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) // perform segmentation on big modifiers val addressesLen: Int = trees.length cfor(0)(_ < addressesLen, _ + 1) { i => - if(trees(i).txs.length > segmentTreshold || trees(i).boxes.length > segmentTreshold) trees ++= trees(i).splitToSegment() + if(trees(i).txs.length > segmentTreshold || trees(i).boxes.length > segmentTreshold) trees ++= trees(i).splitToSegments() } // merge all modifiers to an Array, avoids reallocations durin concatenation (++) @@ -156,57 +156,45 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) general += IndexedErgoTransaction(tx.id, height, globalTxIndex) general += NumericTxIndex(globalTxIndex, tx.id) - //process transaction outputs - cfor(0)(_ < tx.outputs.length, _ + 1) { i => - - val box: ErgoBox = tx.outputs(i) - - boxes += new IndexedErgoBox(Some(height), None, None, box, globalBoxIndex) // box by id - general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number - - globalBoxIndex += 1 - boxCount += 1 - - } - - globalTxIndex += 1 - - } - - // process transactions and boxes by address backwards because the explorer does this - cfor(bt.txs.length - 1)(_ >= 0, _ - 1) { n => - - val tx: ErgoTransaction = bt.txs(n) - tokens.clear() - //process transaction inputs for addresses and tokens + //process transaction inputs if(height != 1) { //only after 1st block (skip genesis box) - cfor(0)(_ < tx.inputs.length, _ + 1) { i => + cfor(0)(_ < tx.inputs.size, _ + 1) { i => val boxIndex: Int = findAndSpendBox(tx.inputs(i).boxId, tx.id, height) - if(boxIndex >= 0) findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(boxes(boxIndex).box.ergoTree)), globalTxIndex - n - 1, Left(boxes(boxIndex).box)) // spend box and add tx + if(boxIndex >= 0) findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(boxes(boxIndex).box.ergoTree)), Left(boxes(boxIndex).box)) // spend box and add tx } } - //process transaction outputs for addresses and tokens - cfor(0)(_ < tx.outputs.length, _ + 1) { i => - + //process transaction outputs + cfor(0)(_ < tx.outputs.size, _ + 1) { i => val box: ErgoBox = tx.outputs(i) + boxes += new IndexedErgoBox(Some(height), None, None, box, globalBoxIndex) // box by id + general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number // box by address - findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)), globalTxIndex - n - 1, Right(boxes(findBox(box.id).get))) + findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)), Right(boxes(findBox(box.id).get))) // check if box is creating a new token, if yes record it if(box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => - if(!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) + if(!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) { general += IndexedTokenSerializer.fromBox(box) + } } + globalBoxIndex += 1 + boxCount += 1 + } + globalTxIndex += 1 + } + // reverses new box and transaction indexes in addresses (needed because the explorer uses this format) + cfor(0)(_ < trees.length, _ + 1) { i => trees(i).finalizeNewTxsAndBoxes() } + log.info(s"Buffered block #$height / $chainHeight [txs: ${bt.txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") if(caughtUp) { diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index e47f80c094..e85eef958b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -28,6 +28,9 @@ case class IndexedErgoAddress(treeHash: ModifierId, override type M = IndexedErgoAddress override def serializer: ScorexSerializer[IndexedErgoAddress] = IndexedErgoAddressSerializer + private val txsTemp: ListBuffer[Long] = ListBuffer.empty[Long] + private val boxesTemp: ListBuffer[Long] = ListBuffer.empty[Long] + private[extra] var boxSegmentCount: Int = 0 private[extra] var txSegmentCount: Int = 0 @@ -71,12 +74,12 @@ case class IndexedErgoAddress(treeHash: ModifierId, } def addTx(tx: Long): IndexedErgoAddress = { - if(txs.last != tx) txs += tx // check for duplicates + if(txsTemp.lastOption.getOrElse(txs.last) != tx) txsTemp.+=:(tx) // check for duplicates this } def addBox(iEb: IndexedErgoBox): IndexedErgoAddress = { - boxes += iEb.globalIndex + boxesTemp.+=:(iEb.globalIndex) balanceInfo.get.add(iEb.box) this } @@ -86,15 +89,22 @@ case class IndexedErgoAddress(treeHash: ModifierId, this } - def splitToSegment(): Array[IndexedErgoAddress] = { + def finalizeNewTxsAndBoxes(): Unit = { + txs ++= txsTemp + boxes ++= boxesTemp + txsTemp.clear() + boxesTemp.clear() + } + + def splitToSegments(): Array[IndexedErgoAddress] = { require(segmentTreshold < txs.length || segmentTreshold < boxes.length, "address does not have enough transactions or boxes for segmentation") val data: ListBuffer[IndexedErgoAddress] = ListBuffer.empty[IndexedErgoAddress] - if(segmentTreshold < txs.length) { + while(txs.length > segmentTreshold) { data += new IndexedErgoAddress(txSegmentId(treeHash, txSegmentCount), txs.take(segmentTreshold), ListBuffer.empty[Long], None) txSegmentCount += 1 txs.remove(0, segmentTreshold) } - if(segmentTreshold < boxes.length) { + while(boxes.length > segmentTreshold) { data += new IndexedErgoAddress(boxSegmentId(treeHash, boxSegmentCount), ListBuffer.empty[Long], boxes.take(segmentTreshold), None) boxSegmentCount += 1 boxes.remove(0, segmentTreshold) From ceb15840ee77ae625b184631b3eac84dc53a49fb Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 18 Sep 2022 16:31:46 +0200 Subject: [PATCH 30/90] updated indexer --- .../nodeView/history/extra/ExtraIndexer.scala | 13 ++++---- .../history/extra/IndexedErgoAddress.scala | 31 +++++++++++++------ .../core/network/PeerConnectionHandler.scala | 2 +- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 7736e21b52..34b13db154 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -2,11 +2,12 @@ package org.ergoplatform.nodeView.history.extra import akka.actor.{Actor, ActorRef, ActorSystem, Props} import org.ergoplatform.ErgoBox.{BoxId, TokenId} -import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} +import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} import org.ergoplatform.modifiers.history.BlockTransactions +import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.ErgoTransaction -import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier +import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.FullBlockApplied import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{GlobalBoxIndexKey, GlobalTxIndexKey, IndexedHeightKey} import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start @@ -193,7 +194,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } // reverses new box and transaction indexes in addresses (needed because the explorer uses this format) - cfor(0)(_ < trees.length, _ + 1) { i => trees(i).finalizeNewTxsAndBoxes() } + cfor(0)(_ < trees.length, _ + 1) { i => trees(i).reverseNewTxsAndBoxes() } log.info(s"Buffered block #$height / $chainHeight [txs: ${bt.txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") @@ -230,14 +231,14 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } override def preStart(): Unit = - context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) + context.system.eventStream.subscribe(self, classOf[FullBlockApplied]) override def postStop(): Unit = log.info(s"Stopped extra indexer at height ${if(lastWroteToDB > 0) lastWroteToDB else indexedHeight}") override def receive: Receive = { - case SemanticallySuccessfulModifier(fb: ErgoFullBlock) if caughtUp => - index(fb.blockTransactions, fb.height) // after the indexer caught up with the chain, stay up to date + case FullBlockApplied(header: Header) if caughtUp => + index(history.typedModifierById[BlockTransactions](header.transactionsId).get, header.height) // after the indexer caught up with the chain, stay up to date case Start(history: ErgoHistory) => _history = history run() diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index e85eef958b..456d625206 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -28,8 +28,8 @@ case class IndexedErgoAddress(treeHash: ModifierId, override type M = IndexedErgoAddress override def serializer: ScorexSerializer[IndexedErgoAddress] = IndexedErgoAddressSerializer - private val txsTemp: ListBuffer[Long] = ListBuffer.empty[Long] - private val boxesTemp: ListBuffer[Long] = ListBuffer.empty[Long] + private var newTxCount: Int = 0 + private var newBoxCount: Int = 0 private[extra] var boxSegmentCount: Int = 0 private[extra] var txSegmentCount: Int = 0 @@ -74,12 +74,16 @@ case class IndexedErgoAddress(treeHash: ModifierId, } def addTx(tx: Long): IndexedErgoAddress = { - if(txsTemp.lastOption.getOrElse(txs.last) != tx) txsTemp.+=:(tx) // check for duplicates + if(txs.last != tx) { // check for duplicates + txs += tx + newTxCount += 1 + } this } def addBox(iEb: IndexedErgoBox): IndexedErgoAddress = { - boxesTemp.+=:(iEb.globalIndex) + boxes += iEb.globalIndex + newBoxCount += 1 balanceInfo.get.add(iEb.box) this } @@ -89,11 +93,20 @@ case class IndexedErgoAddress(treeHash: ModifierId, this } - def finalizeNewTxsAndBoxes(): Unit = { - txs ++= txsTemp - boxes ++= boxesTemp - txsTemp.clear() - boxesTemp.clear() + def reverseNewTxsAndBoxes(): Unit = { + + val arr: ListBuffer[Long] = txs.takeRight(newTxCount).reverse + txs.remove(txs.length - newTxCount, newTxCount) + txs ++= arr + newTxCount = 0 + + arr.clear() + + arr ++= boxes.takeRight(newBoxCount).reverse + boxes.remove(boxes.length - newBoxCount, newBoxCount) + boxes ++= arr + newBoxCount = 0 + } def splitToSegments(): Array[IndexedErgoAddress] = { diff --git a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala index 618ba7656c..641f8f4416 100644 --- a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala +++ b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala @@ -98,7 +98,7 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, handler(handshake) case Failure(t) => - log.info(s"Error during parsing a handshake", t) + log.info(s"Error during parsing a handshake", t.getMessage) //ban the peer for the wrong handshake message //peer will be added to the blacklist and the network controller will send CloseConnection selfPeer.foreach(c => networkControllerRef ! PenalizePeer(c.connectionId.remoteAddress, PenaltyType.PermanentPenalty)) From bc32152b972c62d741a5208096123e165b245ede Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 18 Sep 2022 22:04:53 +0200 Subject: [PATCH 31/90] more optimizations, tx sorting working --- .../nodeView/history/extra/ExtraIndexer.scala | 12 ++-- .../history/extra/IndexedErgoAddress.scala | 64 ++++++++----------- 2 files changed, 34 insertions(+), 42 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 34b13db154..c2f5e96acc 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -83,8 +83,8 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first if(trees(i).treeHash == id) { // address found in last saveLimit modifiers spendOrReceive match { - case Left(box) => trees(i).addTx(globalTxIndex).spendBox(box) // spend box - case Right(iEb) => trees(i).addTx(globalTxIndex).addBox(iEb) // receive box + case Left(box) => trees(i).addTx(box.transactionId,globalTxIndex).spendBox(box) // spend box + case Right(iEb) => trees(i).addTx(iEb.box.transactionId,globalTxIndex).addBox(iEb) // receive box } return } @@ -92,8 +92,8 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) history.typedModifierById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers case Some(x) => spendOrReceive match { - case Left(box) => trees += x.addTx(globalTxIndex).spendBox(box) // spend box - case Right(iEb) => trees += x.addTx(globalTxIndex).addBox(iEb) // receive box + case Left(box) => trees += x.addTx(box.transactionId,globalTxIndex).spendBox(box) // spend box + case Right(iEb) => trees += x.addTx(iEb.box.transactionId,globalTxIndex).addBox(iEb) // receive box } case None => // address not found at all spendOrReceive match { @@ -193,8 +193,8 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } - // reverses new box and transaction indexes in addresses (needed because the explorer uses this format) - cfor(0)(_ < trees.length, _ + 1) { i => trees(i).reverseNewTxsAndBoxes() } + // changes the order of new box and transaction indexes in addresses (needed because the explorer uses this format) + cfor(0)(_ < trees.length, _ + 1) { i => trees(i).sortNewTxsAndBoxes() } log.info(s"Buffered block #$height / $chainHeight [txs: ${bt.txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 456d625206..3abdab322b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -9,17 +9,19 @@ import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{box import org.ergoplatform.settings.Algos import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer -import scorex.util.{ModifierId, bytesToId} +import scorex.util.{ModifierId, ScorexLogging, bytesToId} import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.ErgoTree -import scala.collection.mutable.ListBuffer +import scala.collection.mutable.{ArrayBuffer, ListBuffer} import spire.syntax.all.cfor +import scala.language.postfixOps + case class IndexedErgoAddress(treeHash: ModifierId, txs: ListBuffer[Long], boxes: ListBuffer[Long], - balanceInfo: Option[BalanceInfo]) extends BlockSection { + balanceInfo: Option[BalanceInfo]) extends BlockSection with ScorexLogging { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = fastIdToBytes(treeHash) @@ -28,8 +30,8 @@ case class IndexedErgoAddress(treeHash: ModifierId, override type M = IndexedErgoAddress override def serializer: ScorexSerializer[IndexedErgoAddress] = IndexedErgoAddressSerializer - private var newTxCount: Int = 0 - private var newBoxCount: Int = 0 + private val newTxs: ArrayBuffer[(ModifierId, Long)] = ArrayBuffer.empty[(ModifierId, Long)] + private val newBoxes: ArrayBuffer[Long] = ArrayBuffer.empty[Long] private[extra] var boxSegmentCount: Int = 0 private[extra] var txSegmentCount: Int = 0 @@ -73,56 +75,46 @@ case class IndexedErgoAddress(treeHash: ModifierId, slice(data, offset, limit).toArray } - def addTx(tx: Long): IndexedErgoAddress = { - if(txs.last != tx) { // check for duplicates - txs += tx - newTxCount += 1 - } + private[extra] def addTx(txId: ModifierId, txNum: Long): IndexedErgoAddress = { + if((newTxs.nonEmpty && newTxs(newTxs.length - 1)._2 != txNum) || (txs.nonEmpty && txs.last != txNum)) // check for duplicates + newTxs += Tuple2(txId,txNum) this } - def addBox(iEb: IndexedErgoBox): IndexedErgoAddress = { - boxes += iEb.globalIndex - newBoxCount += 1 + private[extra] def addBox(iEb: IndexedErgoBox): IndexedErgoAddress = { + newBoxes += iEb.globalIndex balanceInfo.get.add(iEb.box) this } - def spendBox(box: ErgoBox): IndexedErgoAddress = { + private[extra] def spendBox(box: ErgoBox): IndexedErgoAddress = { balanceInfo.get.subtract(box) this } - def reverseNewTxsAndBoxes(): Unit = { - - val arr: ListBuffer[Long] = txs.takeRight(newTxCount).reverse - txs.remove(txs.length - newTxCount, newTxCount) - txs ++= arr - newTxCount = 0 - - arr.clear() - - arr ++= boxes.takeRight(newBoxCount).reverse - boxes.remove(boxes.length - newBoxCount, newBoxCount) - boxes ++= arr - newBoxCount = 0 - + private[extra] def sortNewTxsAndBoxes(): Unit = { + txs ++= newTxs.sortBy(_._1).map(_._2) + boxes ++= newBoxes + newTxs.clear() + newBoxes.clear() } - def splitToSegments(): Array[IndexedErgoAddress] = { - require(segmentTreshold < txs.length || segmentTreshold < boxes.length, "address does not have enough transactions or boxes for segmentation") - val data: ListBuffer[IndexedErgoAddress] = ListBuffer.empty[IndexedErgoAddress] - while(txs.length > segmentTreshold) { - data += new IndexedErgoAddress(txSegmentId(treeHash, txSegmentCount), txs.take(segmentTreshold), ListBuffer.empty[Long], None) + private[extra] def splitToSegments(): Array[IndexedErgoAddress] = { + val data: Array[IndexedErgoAddress] = new Array[IndexedErgoAddress]((txs.length / segmentTreshold) + (boxes.length / segmentTreshold)) + var i: Int = 0 + while(txs.length >= segmentTreshold) { + data(i) = new IndexedErgoAddress(txSegmentId(treeHash, txSegmentCount), txs.take(segmentTreshold), ListBuffer.empty[Long], None) + i += 1 txSegmentCount += 1 txs.remove(0, segmentTreshold) } - while(boxes.length > segmentTreshold) { - data += new IndexedErgoAddress(boxSegmentId(treeHash, boxSegmentCount), ListBuffer.empty[Long], boxes.take(segmentTreshold), None) + while(boxes.length >= segmentTreshold) { + data(i) = new IndexedErgoAddress(boxSegmentId(treeHash, boxSegmentCount), ListBuffer.empty[Long], boxes.take(segmentTreshold), None) + i += 1 boxSegmentCount += 1 boxes.remove(0, segmentTreshold) } - data.toArray + data } } From 00b00bfbf144c4a05ff9a0a08a494ec52f98a51d Mon Sep 17 00:00:00 2001 From: jellymlg Date: Mon, 19 Sep 2022 10:57:01 +0200 Subject: [PATCH 32/90] removed unused stuff --- .../nodeView/history/extra/IndexedErgoAddress.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 3abdab322b..d099f32794 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -1,6 +1,6 @@ package org.ergoplatform.nodeView.history.extra -import org.ergoplatform.{ErgoAddress, ErgoBox} +import org.ergoplatform.ErgoBox import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes @@ -16,8 +16,6 @@ import sigmastate.Values.ErgoTree import scala.collection.mutable.{ArrayBuffer, ListBuffer} import spire.syntax.all.cfor -import scala.language.postfixOps - case class IndexedErgoAddress(treeHash: ModifierId, txs: ListBuffer[Long], boxes: ListBuffer[Long], @@ -121,7 +119,6 @@ case class IndexedErgoAddress(treeHash: ModifierId, object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] { def hashErgoTree(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) - def hashAddress(address: ErgoAddress): Array[Byte] = hashErgoTree(address.script) def boxSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " box segment " + segmentNum)) def txSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " tx segment " + segmentNum)) From 3ff45cb6b349047fc3b7ab0e6cb402e63fcef712 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Mon, 26 Sep 2022 23:21:49 +0200 Subject: [PATCH 33/90] reverted sorting changes --- .../nodeView/history/extra/ExtraIndexer.scala | 11 ++++------- .../history/extra/IndexedErgoAddress.scala | 19 ++++--------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index c2f5e96acc..65ec099c89 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -83,8 +83,8 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first if(trees(i).treeHash == id) { // address found in last saveLimit modifiers spendOrReceive match { - case Left(box) => trees(i).addTx(box.transactionId,globalTxIndex).spendBox(box) // spend box - case Right(iEb) => trees(i).addTx(iEb.box.transactionId,globalTxIndex).addBox(iEb) // receive box + case Left(box) => trees(i).addTx(globalTxIndex).spendBox(box) // spend box + case Right(iEb) => trees(i).addTx(globalTxIndex).addBox(iEb) // receive box } return } @@ -92,8 +92,8 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) history.typedModifierById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers case Some(x) => spendOrReceive match { - case Left(box) => trees += x.addTx(box.transactionId,globalTxIndex).spendBox(box) // spend box - case Right(iEb) => trees += x.addTx(iEb.box.transactionId,globalTxIndex).addBox(iEb) // receive box + case Left(box) => trees += x.addTx(globalTxIndex).spendBox(box) // spend box + case Right(iEb) => trees += x.addTx(globalTxIndex).addBox(iEb) // receive box } case None => // address not found at all spendOrReceive match { @@ -193,9 +193,6 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } - // changes the order of new box and transaction indexes in addresses (needed because the explorer uses this format) - cfor(0)(_ < trees.length, _ + 1) { i => trees(i).sortNewTxsAndBoxes() } - log.info(s"Buffered block #$height / $chainHeight [txs: ${bt.txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") if(caughtUp) { diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index d099f32794..ea5625ee10 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -13,7 +13,7 @@ import scorex.util.{ModifierId, ScorexLogging, bytesToId} import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.ErgoTree -import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import scala.collection.mutable.ListBuffer import spire.syntax.all.cfor case class IndexedErgoAddress(treeHash: ModifierId, @@ -28,9 +28,6 @@ case class IndexedErgoAddress(treeHash: ModifierId, override type M = IndexedErgoAddress override def serializer: ScorexSerializer[IndexedErgoAddress] = IndexedErgoAddressSerializer - private val newTxs: ArrayBuffer[(ModifierId, Long)] = ArrayBuffer.empty[(ModifierId, Long)] - private val newBoxes: ArrayBuffer[Long] = ArrayBuffer.empty[Long] - private[extra] var boxSegmentCount: Int = 0 private[extra] var txSegmentCount: Int = 0 @@ -73,14 +70,13 @@ case class IndexedErgoAddress(treeHash: ModifierId, slice(data, offset, limit).toArray } - private[extra] def addTx(txId: ModifierId, txNum: Long): IndexedErgoAddress = { - if((newTxs.nonEmpty && newTxs(newTxs.length - 1)._2 != txNum) || (txs.nonEmpty && txs.last != txNum)) // check for duplicates - newTxs += Tuple2(txId,txNum) + private[extra] def addTx(tx: Long): IndexedErgoAddress = { + if(txs.lastOption.getOrElse(-1) != tx) txs += tx // check for duplicates this } private[extra] def addBox(iEb: IndexedErgoBox): IndexedErgoAddress = { - newBoxes += iEb.globalIndex + boxes += iEb.globalIndex balanceInfo.get.add(iEb.box) this } @@ -90,13 +86,6 @@ case class IndexedErgoAddress(treeHash: ModifierId, this } - private[extra] def sortNewTxsAndBoxes(): Unit = { - txs ++= newTxs.sortBy(_._1).map(_._2) - boxes ++= newBoxes - newTxs.clear() - newBoxes.clear() - } - private[extra] def splitToSegments(): Array[IndexedErgoAddress] = { val data: Array[IndexedErgoAddress] = new Array[IndexedErgoAddress]((txs.length / segmentTreshold) + (boxes.length / segmentTreshold)) var i: Int = 0 From e638c63465284264a835f1bd948ed07cbf3e8437 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 2 Oct 2022 20:12:42 +0200 Subject: [PATCH 34/90] tx sorting done, boxes seem good --- .../scala/org/ergoplatform/http/api/BlockchainApiRoute.scala | 2 +- .../nodeView/history/extra/IndexedErgoAddress.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index d4f84e3c2f..c0def368e6 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -93,7 +93,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getTxsByAddress(addr: ErgoAddress, offset: Int, limit: Int): Future[(Seq[IndexedErgoTransaction],Long)] = getHistory.map { history => getAddress(addr)(history) match { - case Some(addr) => (addr.retrieveTxs(history, offset, limit).reverse, addr.txCount()) + case Some(addr) => (addr.retrieveTxs(history, offset, limit), addr.txCount()) case None => (Seq.empty[IndexedErgoTransaction], 0L) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index ea5625ee10..1a160edbe1 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -151,8 +151,8 @@ object IndexedErgoAddress { def slice[T](arr: Iterable[T], offset: Int, limit: Int): Iterable[T] = arr.slice(arr.size - offset - limit, arr.size - offset) - def getTxs(arr: Iterable[Long])(history: ErgoHistoryReader): Array[IndexedErgoTransaction] = - arr.map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray + def getTxs(arr: Iterable[Long])(history: ErgoHistoryReader): Array[IndexedErgoTransaction] = // sorted to match explorer + arr.map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray.sortBy(tx => (-tx.height, tx.id)) def getBoxes(arr: Iterable[Long])(history: ErgoHistoryReader): Array[IndexedErgoBox] = arr.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray From 248c6bdd6a817d2f0bd29484a0e8ab56009fc0a2 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Mon, 3 Oct 2022 00:00:23 +0200 Subject: [PATCH 35/90] fixed comment --- src/main/resources/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 07c9896679..ea228d1e99 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -74,7 +74,7 @@ ergo { # Minimal fee amount of transactions mempool accepts minimalFeeAmount = 1000000 - # If true, the node will store all transactions, boxes and addresses. + # If true, the node will store all transactions, boxes and addresses in an index. extraIndex = false # List with hex-encoded identifiers of transactions banned from getting into memory pool From e7b5fe8ec20375b1c1c467cf6bac3c0fb9224d1b Mon Sep 17 00:00:00 2001 From: jellymlg Date: Mon, 3 Oct 2022 00:19:10 +0200 Subject: [PATCH 36/90] fixed tests --- .../scala/org/ergoplatform/db/KvStoreReaderSpec.scala | 6 +++--- .../scala/org/ergoplatform/db/LDBKVStoreSpec.scala | 10 +++++----- .../history/BlockSectionValidationSpecification.scala | 6 +++--- .../nodeView/history/storage/HistoryStorageSpec.scala | 11 +++++++---- .../wallet/persistence/WalletStorageSpec.scala | 2 +- .../scala/org/ergoplatform/tools/ChainGenerator.scala | 2 +- .../org/ergoplatform/utils/HistoryTestHelpers.scala | 2 +- src/test/scala/org/ergoplatform/utils/Stubs.scala | 2 +- 8 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala b/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala index b230d6d7c0..c201ebd15d 100644 --- a/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala +++ b/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala @@ -11,10 +11,10 @@ class KvStoreReaderSpec extends AnyPropSpec with Matchers with DBSpec { val keyEnd = byteString("Z") store.getRange(keyStart, keyEnd).length shouldBe 0 - store.insert(Seq(keyStart -> keyStart)).get + store.insert(Array(keyStart -> keyStart)).get store.getRange(keyStart, keyEnd).length shouldBe 1 - store.insert(Seq(keyEnd -> keyEnd)).get + store.insert(Array(keyEnd -> keyEnd)).get store.getRange(keyStart, keyEnd).length shouldBe 2 // keys before the range @@ -27,7 +27,7 @@ class KvStoreReaderSpec extends AnyPropSpec with Matchers with DBSpec { store.getRange(byteString("<"), byteString("?")).length shouldBe 0 //removing keys - store.remove(Seq(keyStart, keyEnd)).get + store.remove(Array(keyStart, keyEnd)).get store.getRange(keyStart, keyEnd).length shouldBe 0 } } diff --git a/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala b/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala index 0271607ec3..cf675c916d 100644 --- a/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala +++ b/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala @@ -10,14 +10,14 @@ class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { val valueA = (byteString("A"), byteString("1")) val valueB = (byteString("B"), byteString("2")) - store.update(toInsert = Seq(valueA, valueB), toRemove = Seq.empty).get + store.update(toInsert = Array(valueA, valueB), toRemove = Array.empty).get store.get(valueA._1).toBs shouldBe Some(valueA._2).toBs store.get(valueB._1).toBs shouldBe Some(valueB._2).toBs store.getAll.toSeq.toBs shouldBe Seq(valueA, valueB).toBs - store.update(toInsert = Seq.empty, toRemove = Seq(valueA._1)).get + store.update(toInsert = Array.empty, toRemove = Array(valueA._1)).get store.get(valueA._1) shouldBe None } } @@ -28,11 +28,11 @@ class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { val valA = byteString("1") val valB = byteString("2") - store.insert(Seq(key -> valA)).get + store.insert(Array(key -> valA)).get store.get(key).toBs shouldBe Some(valA).toBs - store.insert(Seq(key -> valB)).get + store.insert(Array(key -> valB)).get store.get(key).toBs shouldBe Some(valB).toBs @@ -49,7 +49,7 @@ class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { val valueE = (byteString("E"), byteString("3")) val valueF = (byteString("F"), byteString("4")) - store.insert(Seq(valueA, valueB, valueC, valueD, valueE, valueF)).get + store.insert(Array(valueA, valueB, valueC, valueD, valueE, valueF)).get store.lastKeyInRange(valueA._1, valueC._1).get.toSeq shouldBe valueC._1.toSeq store.lastKeyInRange(valueD._1, valueF._1).get.toSeq shouldBe valueF._1.toSeq diff --git a/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala index d4290b4ba6..a059e416e9 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala @@ -1,6 +1,6 @@ package org.ergoplatform.nodeView.history -import org.ergoplatform.modifiers.NonHeaderBlockSection +import org.ergoplatform.modifiers.{BlockSection, NonHeaderBlockSection} import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.Header @@ -79,10 +79,10 @@ class BlockSectionValidationSpecification extends HistoryTestHelpers { // should not be able to apply if corresponding header is marked as invalid history.applicableTry(section) shouldBe 'success - history.historyStorage.insert(Seq(history.validityKey(header.id) -> Array(0.toByte)), Seq.empty).get + history.historyStorage.insert(Array(history.validityKey(header.id) -> Array(0.toByte)), Array.empty[BlockSection]).get history.isSemanticallyValid(header.id) shouldBe ModifierSemanticValidity.Invalid history.applicableTry(section) shouldBe 'failure - history.historyStorage.insert(Seq(history.validityKey(header.id) -> Array(1.toByte)), Seq.empty).get + history.historyStorage.insert(Array(history.validityKey(header.id) -> Array(1.toByte)), Array.empty[BlockSection]).get // should not be able to apply if already in history history.applicableTry(section) shouldBe 'success diff --git a/src/test/scala/org/ergoplatform/nodeView/history/storage/HistoryStorageSpec.scala b/src/test/scala/org/ergoplatform/nodeView/history/storage/HistoryStorageSpec.scala index 6510d46b2f..7020d1fee7 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/storage/HistoryStorageSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/storage/HistoryStorageSpec.scala @@ -1,5 +1,8 @@ package org.ergoplatform.nodeView.history.storage +import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.modifiers.history.ADProofs +import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.settings.Algos import org.ergoplatform.utils.HistoryTestHelpers @@ -12,11 +15,11 @@ class HistoryStorageSpec extends HistoryTestHelpers { val db = HistoryStorage(settings) property("Write Read Remove") { - val headers = Gen.listOfN(20, defaultHeaderGen).sample.get - val modifiers = Gen.listOfN(20, randomADProofsGen).sample.get + val headers: Array[Header] = Gen.listOfN(20, defaultHeaderGen).sample.get.toArray + val modifiers: Array[ADProofs] = Gen.listOfN(20, randomADProofsGen).sample.get.toArray def validityKey(id: ModifierId) = ByteArrayWrapper(Algos.hash("validity".getBytes(ErgoHistory.CharsetName) ++ idToBytes(id))) - val indexes = headers.flatMap(h => Seq(validityKey(h.id) -> Array(1.toByte))) - db.insert(indexes, headers ++ modifiers) shouldBe 'success + val indexes = headers.flatMap(h => Array(validityKey(h.id) -> Array(1.toByte))) + db.insert(indexes, (headers ++ modifiers).asInstanceOf[Array[BlockSection]]) shouldBe 'success headers.forall(h => db.contains(h.id)) shouldBe true modifiers.forall(m => db.contains(m.id)) shouldBe true diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala index 552e0bf270..a78cb1dd4d 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala @@ -27,7 +27,7 @@ class WalletStorageSpec val bytes = DerivationPathSerializer.toBytes(path) acc ++ Ints.toByteArray(bytes.length) ++ bytes } - store.insert(Seq(SecretPathsKey -> toInsert)).get + store.insert(Array(SecretPathsKey -> toInsert)).get } forAll(Gen.nonEmptyListOf(derivationPathGen)) { paths => diff --git a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala index 789a7b06fc..9d168632ec 100644 --- a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala +++ b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala @@ -62,7 +62,7 @@ object ChainGenerator extends App with ErgoTestHelpers { -1, poPoWBootstrap = false, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 20, - 1000000, 100, adProofsSuffixLength = 112*1024) + 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false) val ms = settings.chainSettings.monetary.copy( minerRewardDelay = RewardDelay ) diff --git a/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala b/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala index 628090c90a..bfc67cc374 100644 --- a/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala +++ b/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala @@ -49,7 +49,7 @@ trait HistoryTestHelpers extends ErgoPropertyTest { PoPoWBootstrap, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, - rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024 + rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false ) val scorexSettings: ScorexSettings = null val walletSettings: WalletSettings = null diff --git a/src/test/scala/org/ergoplatform/utils/Stubs.scala b/src/test/scala/org/ergoplatform/utils/Stubs.scala index c7d0139081..305279f84e 100644 --- a/src/test/scala/org/ergoplatform/utils/Stubs.scala +++ b/src/test/scala/org/ergoplatform/utils/Stubs.scala @@ -369,7 +369,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with PoPoWBootstrap, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, internalMinersCount = 1, internalMinerPollingInterval = 1.second,miningPubKeyHex = None, offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, - rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024 + rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false ) val scorexSettings: ScorexSettings = null val walletSettings: WalletSettings = null From 0468c1c6d913dd572b6a64e48941aedb0ce080a5 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Mon, 3 Oct 2022 15:04:52 +0200 Subject: [PATCH 37/90] added isExplorer flag in /info --- .../org/ergoplatform/local/ErgoStatsCollector.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala b/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala index bb41748619..f42677a858 100644 --- a/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala +++ b/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala @@ -69,7 +69,8 @@ class ErgoStatsCollector(readersHolder: ActorRef, None, LaunchParameters, eip27Supported = true, - settings.scorexSettings.restApi.publicUrl) + settings.scorexSettings.restApi.publicUrl, + settings.nodeSettings.extraIndex) override def receive: Receive = onConnectedPeers orElse @@ -171,12 +172,14 @@ object ErgoStatsCollector { * @param headersScore - cumulative difficulty of best headers-chain * @param bestFullBlockOpt - best full-block id (header id of such block) * @param fullBlocksScore - cumulative difficulty of best full blocks chain + * @param maxPeerHeight - maximum block height of connected peers * @param launchTime - when the node was launched (in Java time format, basically, UNIX time * 1000) * @param lastIncomingMessageTime - when the node received last p2p message (in Java time) * @param genesisBlockIdOpt - header id of genesis block * @param parameters - array with network parameters at the moment * @param eip27Supported - whether EIP-27 locked in - * @param restApiUrl publicly accessible url of node which exposes restApi in firewall + * @param restApiUrl - publicly accessible url of node which exposes restApi in firewall + * @param extraIndex - whether the node has additional indexing enabled */ case class NodeInfo(nodeName: String, appVersion: String, @@ -191,13 +194,14 @@ object ErgoStatsCollector { headersScore: Option[BigInt], bestFullBlockOpt: Option[ErgoFullBlock], fullBlocksScore: Option[BigInt], - maxPeerHeight : Option[Int], // Maximum block height of connected peers + maxPeerHeight : Option[Int], launchTime: Long, lastIncomingMessageTime: Long, genesisBlockIdOpt: Option[String], parameters: Parameters, eip27Supported: Boolean, - restApiUrl: Option[URL]) + restApiUrl: Option[URL], + extraIndex: Boolean) object NodeInfo extends ApiCodecs { implicit val paramsEncoder: Encoder[Parameters] = org.ergoplatform.settings.ParametersSerializer.jsonEncoder @@ -223,6 +227,7 @@ object ErgoStatsCollector { "stateType" -> ni.stateType.stateTypeName.asJson, "stateVersion" -> ni.stateVersion.asJson, "isMining" -> ni.isMining.asJson, + "isExplorer" -> ni.extraIndex.asJson, "peersCount" -> ni.peersCount.asJson, "launchTime" -> ni.launchTime.asJson, "lastSeenMessageTime" -> ni.lastIncomingMessageTime.asJson, From 265e087615ac3ca48c8f70ea5938e8f5cf43fb0c Mon Sep 17 00:00:00 2001 From: jellymlg Date: Wed, 12 Oct 2022 18:16:42 +0200 Subject: [PATCH 38/90] changed some API endpoints to POST, so large addresses/ergotrees are now working properly --- src/main/resources/api/openapi.yaml | 107 +++++++++--------- .../http/api/BlockchainApiRoute.scala | 12 +- .../http/api/ErgoBaseApiRoute.scala | 4 +- 3 files changed, 64 insertions(+), 59 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 65f2104f94..ab02ba646a 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -5324,19 +5324,20 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /blockchain/transaction/byAddress/{address}: - get: + /blockchain/transaction/byAddress: + post: summary: Retrieve transactions by their associated address operationId: getTxsByAddress tags: - blockchain + requestBody: + required: true + content: + application/json: + description: adderess associated with transactions + schema: + $ref: '#/components/schemas/ErgoAddress' parameters: - - in: path - name: address - required: true - description: adderess associated with transactions - schema: - $ref: '#/components/schemas/ErgoAddress' - in: query name: offset required: false @@ -5488,19 +5489,20 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /blockchain/box/byAddress/{address}: - get: + /blockchain/box/byAddress: + post: summary: Retrieve boxes by their associated address operationId: getBoxesByAddress tags: - blockchain + requestBody: + required: true + content: + application/json: + description: adderess associated with boxes + schema: + $ref: '#/components/schemas/ErgoAddress' parameters: - - in: path - name: address - required: true - description: adderess associated with boxes - schema: - $ref: '#/components/schemas/ErgoAddress' - in: query name: offset required: false @@ -5546,19 +5548,20 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /blockchain/box/unspent/byAddress/{address}: - get: + /blockchain/box/unspent/byAddress: + post: summary: Retrieve unspent boxes by their associated address operationId: getBoxesByAddressUnspent tags: - blockchain + requestBody: + required: true + content: + application/json: + description: adderess associated with unspent boxes + schema: + $ref: '#/components/schemas/ErgoAddress' parameters: - - in: path - name: address - required: true - description: adderess associated with unspent boxes - schema: - $ref: '#/components/schemas/ErgoAddress' - in: query name: offset required: false @@ -5638,20 +5641,21 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /blockchain/box/byErgoTree/{ergoTreeHex}: - get: + /blockchain/box/byErgoTree: + post: summary: Retrieve boxes by their associated ergotree operationId: getBoxesByErgoTree tags: - blockchain + requestBody: + required: true + content: + application/json: + description: hex encoded ergotree + schema: + type: string + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' parameters: - - in: path - name: ergoTreeHex - required: true - description: hex encoded ergotree - schema: - type: string - example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' - in: query name: offset required: false @@ -5691,20 +5695,21 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /blockchain/box/unspent/byErgoTree/{ergoTreeHex}: - get: + /blockchain/box/unspent/byErgoTree: + post: summary: Retrieve unspent boxes by their associated ergotree operationId: getBoxesByErgoTreeUnspent tags: - blockchain + requestBody: + required: true + content: + application/json: + description: hex encoded ergotree + schema: + type: string + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' parameters: - - in: path - name: ergoTreeHex - required: true - description: hex encoded ergotree - schema: - type: string - example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' - in: query name: offset required: false @@ -5774,19 +5779,19 @@ paths: schema: $ref: '#/components/schemas/ApiError' - /blockchain/balance/{address}: - get: + /blockchain/balance: + post: summary: Retrieve confirmed and unconfirmed balance of an address operationId: getAddressBalanceTotal tags: - blockchain - parameters: - - in: path - name: address - required: true - description: adderess with balance - schema: - $ref: '#/components/schemas/ErgoAddress' + requestBody: + required: true + content: + application/json: + description: adderess with balance + schema: + $ref: '#/components/schemas/ErgoAddress' responses: '200': description: balance information diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index c0def368e6..9726cd7607 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -98,7 +98,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getTxsByAddressR: Route = (get & pathPrefix("transaction" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => + private def getTxsByAddressR: Route = (post & pathPrefix("transaction" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => if(limit > MaxItems) { BadRequest(s"No more than $MaxItems transactions can be requested") }else { @@ -156,7 +156,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByAddressR: Route = (get & pathPrefix("box" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => + private def getBoxesByAddressR: Route = (post & pathPrefix("box" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => if(limit > MaxItems) { BadRequest(s"No more than $MaxItems boxes can be requested") }else { @@ -172,7 +172,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByAddressUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => + private def getBoxesByAddressUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging) { (address, offset, limit) => if(limit > MaxItems) { BadRequest(s"No more than $MaxItems boxes can be requested") }else { @@ -209,7 +209,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByErgoTreeR: Route = (get & pathPrefix("box" / "byErgoTree") & ergoTree & paging) { (tree, offset, limit) => + private def getBoxesByErgoTreeR: Route = (post & pathPrefix("box" / "byErgoTree") & ergoTree & paging) { (tree, offset, limit) => if(limit > MaxItems) { BadRequest(s"No more than $MaxItems boxes can be requested") }else { @@ -225,7 +225,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getBoxesByErgoTreeUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byErgoTree") & ergoTree & paging) { (tree, offset, limit) => + private def getBoxesByErgoTreeUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byErgoTree") & ergoTree & paging) { (tree, offset, limit) => if(limit > MaxItems) { BadRequest(s"No more than $MaxItems boxes can be requested") }else { @@ -264,7 +264,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } - private def getAddressBalanceTotalR: Route = (get & pathPrefix("balance") & ergoAddress) { address => + private def getAddressBalanceTotalR: Route = (post & pathPrefix("balance") & ergoAddress) { address => ApiResponse(getAddressBalanceTotal(address)) } diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala index 8625441fbb..56571fe84a 100644 --- a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala @@ -40,7 +40,7 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { } } - val ergoAddress: Directive1[ErgoAddress] = pathPrefix(Segment).flatMap(handleErgoAddress) + val ergoAddress: Directive1[ErgoAddress] = entity(as[String]).flatMap(handleErgoAddress) private def handleErgoAddress(value: String): Directive1[ErgoAddress] = { ae.fromString(value) match { @@ -49,7 +49,7 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { } } - val ergoTree: Directive1[ErgoTree] = pathPrefix(Segment).flatMap(handleErgoTree) + val ergoTree: Directive1[ErgoTree] = entity(as[String]).flatMap(handleErgoTree) private def handleErgoTree(value: String): Directive1[ErgoTree] = { Base16.decode(value) match { From 4ad2fde8a461d4a98e00c15f84a5bcf5e4b2a581 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Fri, 21 Oct 2022 19:09:54 +0200 Subject: [PATCH 39/90] removed null --- .../ergoplatform/http/api/BlockchainApiRoute.scala | 13 +++++++++++-- .../ergoplatform/http/api/ErgoBaseApiRoute.scala | 12 ------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 9726cd7607..c80906d7f1 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -1,7 +1,7 @@ package org.ergoplatform.http.api import akka.actor.{ActorRef, ActorRefFactory} -import akka.http.scaladsl.server.{Directive, Route} +import akka.http.scaladsl.server.{Directive, Directive1, Route, ValidationRejection} import akka.pattern.ask import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} import org.ergoplatform.nodeView.ErgoReadersHolder.{GetDataFromHistory, GetReaders, Readers} @@ -19,6 +19,7 @@ import spire.implicits.cfor import java.nio.ByteBuffer import scala.concurrent.Future +import scala.util.Success case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSettings) (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute with ApiCodecs { @@ -29,8 +30,16 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private val MaxItems = 16384 - override val ae: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder + implicit val ae: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder + val ergoAddress: Directive1[ErgoAddress] = entity(as[String]).flatMap(handleErgoAddress) + + private def handleErgoAddress(value: String): Directive1[ErgoAddress] = { + ae.fromString(value) match { + case Success(addr) => provide(addr) + case _ => reject(ValidationRejection("Wrong address format")) + } + } override val route: Route = pathPrefix("blockchain") { getTxByIdR ~ diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala index 56571fe84a..5c00e87902 100644 --- a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala @@ -11,7 +11,6 @@ import org.ergoplatform.settings.{Algos, ErgoSettings} import scorex.core.api.http.{ApiError, ApiRoute} import scorex.util.{ModifierId, bytesToId} import akka.pattern.ask -import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedTransaction import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome._ @@ -26,8 +25,6 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { implicit val ec: ExecutionContextExecutor = context.dispatcher - implicit val ae: ErgoAddressEncoder = null - val modifierId: Directive1[ModifierId] = pathPrefix(Segment).flatMap(handleModifierId) val modifierIdGet: Directive1[ModifierId] = parameters("id".as[String]) @@ -40,15 +37,6 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { } } - val ergoAddress: Directive1[ErgoAddress] = entity(as[String]).flatMap(handleErgoAddress) - - private def handleErgoAddress(value: String): Directive1[ErgoAddress] = { - ae.fromString(value) match { - case Success(addr) => provide(addr) - case _ => reject(ValidationRejection("Wrong address format")) - } - } - val ergoTree: Directive1[ErgoTree] = entity(as[String]).flatMap(handleErgoTree) private def handleErgoTree(value: String): Directive1[ErgoTree] = { From d789d74b44fdaf6e63cbe2b5eae7ffe2ccb5bbdc Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sat, 22 Oct 2022 02:02:25 +0200 Subject: [PATCH 40/90] added scaladoc, separate db methods and serializer class --- .../org/ergoplatform/db/LDBStoreBench.scala | 4 +- .../org/ergoplatform/http/api/ApiCodecs.scala | 2 +- .../http/api/BlockchainApiRoute.scala | 16 +-- .../history/HistoryModifierSerializer.scala | 31 ------ .../nodeView/history/ErgoHistoryReader.scala | 14 ++- .../nodeView/history/extra/BalanceInfo.scala | 2 +- .../history/extra/ExtraIndexSerializer.scala | 51 +++++++++ .../nodeView/history/extra/ExtraIndexer.scala | 74 +++++++++++-- .../history/extra/IndexedErgoAddress.scala | 103 +++++++++++++++++- .../history/extra/IndexedErgoBox.scala | 30 +++-- .../extra/IndexedErgoTransaction.scala | 18 ++- .../nodeView/history/extra/IndexedToken.scala | 36 +++++- .../nodeView/history/extra/NumericIndex.scala | 41 ++++++- .../history/storage/HistoryStorage.scala | 48 +++++--- 14 files changed, 374 insertions(+), 96 deletions(-) create mode 100644 src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala diff --git a/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala b/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala index 2467856d46..652bb40982 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala @@ -37,7 +37,7 @@ object LDBStoreBench } val txsWithDbGen: Gen[(Seq[BlockTransactions], LDBKVStore)] = txsGen.map { bts => - val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes) + val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes).toArray val db = storeLDB() toInsert.grouped(5).foreach(db.insert(_).get) bts -> storeLDB @@ -53,7 +53,7 @@ object LDBStoreBench private def randomVersion: Digest32 = Algos.hash(Longs.toByteArray(Random.nextLong())) private def benchWriteLDB(bts: Seq[BlockTransactions]): Unit = { - val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes) + val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes).toArray val db = storeLDB() toInsert.grouped(5).foreach(db.insert(_).get) } diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index 32253db9c6..ee1278bda2 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -473,7 +473,7 @@ trait ApiCodecs extends JsonCodecs { implicit val indexedBoxEncoder: Encoder[IndexedErgoBox] = { iEb => iEb.box.asJson.deepMerge(Json.obj( "globalIndex" -> iEb.globalIndex.asJson, - "inclusionHeight" -> iEb.inclusionHeightOpt.asJson, + "inclusionHeight" -> iEb.inclusionHeight.asJson, "address" -> ExtraIndexerRef.getAddressEncoder.toString(iEb.getAddress).asJson, "spentTransactionId" -> iEb.spendingTxIdOpt.asJson )) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index c80906d7f1..34708c75a6 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -64,13 +64,13 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting (readersHolder ? GetReaders).mapTo[Readers].map(r => (r.h, r.m)) private def getAddress(tree: ErgoTree)(history: ErgoHistoryReader): Option[IndexedErgoAddress] = { - history.typedModifierById[IndexedErgoAddress](bytesToId(IndexedErgoAddressSerializer.hashErgoTree(tree))) + history.typedExtraIndexById[IndexedErgoAddress](bytesToId(IndexedErgoAddressSerializer.hashErgoTree(tree))) } private def getAddress(addr: ErgoAddress)(history: ErgoHistoryReader): Option[IndexedErgoAddress] = getAddress(addr.script)(history) private def getTxById(id: ModifierId)(history: ErgoHistoryReader): Option[IndexedErgoTransaction] = - history.typedModifierById[IndexedErgoTransaction](id) match { + history.typedExtraIndexById[IndexedErgoTransaction](id) match { case Some(tx) => Some(tx.retrieveBody(history)) case None => None } @@ -85,7 +85,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } private def getTxByIndex(index: Long)(history: ErgoHistoryReader): Option[IndexedErgoTransaction] = - getTxById(history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(index))).get.m)(history) + getTxById(history.typedExtraIndexById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(index))).get.m)(history) private def getLastTx(history: ErgoHistoryReader): IndexedErgoTransaction = getTxByIndex(ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalTxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong - 1)(history).get @@ -120,7 +120,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting val base: Int = getLastTx(history).globalIndex.toInt - offset val txIds: Array[ModifierId] = new Array[ModifierId](limit) cfor(0)(_ < limit, _ + 1) { i => - txIds(i) = history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(base - limit + i))).get.m + txIds(i) = history.typedExtraIndexById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(base - limit + i))).get.m } txIds.reverse } @@ -134,7 +134,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } private def getBoxById(id: ModifierId)(history: ErgoHistoryReader): Option[IndexedErgoBox] = - history.typedModifierById[IndexedErgoBox](id) + history.typedExtraIndexById[IndexedErgoBox](id) private def getBoxByIdF(id: ModifierId): Future[Option[IndexedErgoBox]] = getHistory.map { history => @@ -146,7 +146,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } private def getBoxByIndex(index: Long)(history: ErgoHistoryReader): Option[IndexedErgoBox] = - getBoxById(history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(index))).get.m)(history) + getBoxById(history.typedExtraIndexById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(index))).get.m)(history) private def getBoxByIndexF(index: Long): Future[Option[IndexedErgoBox]] = getHistory.map { history => @@ -197,7 +197,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting val base: Int = getLastBox(history).globalIndex.toInt - offset val boxIds: Array[ModifierId] = new Array[ModifierId](limit) cfor(0)(_ < limit, _ + 1) { i => - boxIds(i) = history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(base - limit + i))).get.m + boxIds(i) = history.typedExtraIndexById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(base - limit + i))).get.m } boxIds.reverse } @@ -244,7 +244,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getTokenInfoById(id: ModifierId): Future[Option[IndexedToken]] = { getHistory.map { history => - history.typedModifierById[IndexedToken](IndexedTokenSerializer.uniqueId(id)) + history.typedExtraIndexById[IndexedToken](IndexedTokenSerializer.uniqueId(id)) } } diff --git a/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala b/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala index d132eccadb..d1ef4eb208 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/HistoryModifierSerializer.scala @@ -3,7 +3,6 @@ package org.ergoplatform.modifiers.history import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionSerializer} import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} -import org.ergoplatform.nodeView.history.extra.{IndexedErgoAddress, IndexedErgoAddressSerializer, IndexedErgoBox, IndexedErgoBoxSerializer, IndexedErgoTransaction, IndexedErgoTransactionSerializer, IndexedToken, IndexedTokenSerializer, NumericBoxIndex, NumericBoxIndexSerializer, NumericTxIndex, NumericTxIndexSerializer} import scorex.core.serialization.ScorexSerializer import scorex.util.serialization.{Reader, Writer} @@ -23,24 +22,6 @@ object HistoryModifierSerializer extends ScorexSerializer[BlockSection] { case m: Extension => w.put(Extension.modifierTypeId) ExtensionSerializer.serialize(m, w) - case m: IndexedErgoAddress => - w.put(IndexedErgoAddress.modifierTypeId) - IndexedErgoAddressSerializer.serialize(m, w) - case m: IndexedErgoTransaction => - w.put(IndexedErgoTransaction.modifierTypeId) - IndexedErgoTransactionSerializer.serialize(m, w) - case m: IndexedErgoBox => - w.put(IndexedErgoBox.modifierTypeId) - IndexedErgoBoxSerializer.serialize(m, w) - case m: NumericTxIndex => - w.put(NumericTxIndex.modifierTypeId) - NumericTxIndexSerializer.serialize(m, w) - case m: NumericBoxIndex => - w.put(NumericBoxIndex.modifierTypeId) - NumericBoxIndexSerializer.serialize(m, w) - case m: IndexedToken => - w.put(IndexedToken.modifierTypeId) - IndexedTokenSerializer.serialize(m, w) case m => throw new Error(s"Serialization for unknown modifier: $m") } @@ -56,18 +37,6 @@ object HistoryModifierSerializer extends ScorexSerializer[BlockSection] { BlockTransactionsSerializer.parse(r) case Extension.`modifierTypeId` => ExtensionSerializer.parse(r) - case IndexedErgoAddress.`modifierTypeId` => - IndexedErgoAddressSerializer.parse(r) - case IndexedErgoTransaction.`modifierTypeId` => - IndexedErgoTransactionSerializer.parse(r) - case IndexedErgoBox.`modifierTypeId` => - IndexedErgoBoxSerializer.parse(r) - case NumericTxIndex.`modifierTypeId` => - NumericTxIndexSerializer.parse(r) - case NumericBoxIndex.`modifierTypeId` => - NumericBoxIndexSerializer.parse(r) - case IndexedToken.`modifierTypeId` => - IndexedTokenSerializer.parse(r) case m => throw new Error(s"Deserialization for unknown type byte: $m") } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala index 0637f04cdd..92ef990c0c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala @@ -7,6 +7,7 @@ import org.ergoplatform.modifiers.history.popow.{NipopowAlgos, NipopowProof, PoP import org.ergoplatform.modifiers.state.UTXOSnapshotChunk import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NonHeaderBlockSection} import org.ergoplatform.nodeView.history.ErgoHistory.Height +import org.ergoplatform.nodeView.history.extra.ExtraIndex import org.ergoplatform.nodeView.history.storage._ import org.ergoplatform.nodeView.history.storage.modifierprocessors._ import org.ergoplatform.nodeView.history.storage.modifierprocessors.popow.PoPoWProofsProcessor @@ -99,13 +100,24 @@ trait ErgoHistoryReader * * @param id - modifier id * @tparam T - expected Type - * @return semantically valid ErgoPersistentModifier of type T with the given id it is in history + * @return semantically valid ErgoPersistentModifier of type T with the given id if it is in history */ def typedModifierById[T <: BlockSection : ClassTag](id: ModifierId): Option[T] = modifierById(id) match { case Some(m: T) => Some(m) case _ => None } + /** Get index of expected type by its identifier + * @param id - index id + * @tparam T - expected Type + * @return index of type T with the given id if it is in history + */ + def typedExtraIndexById[T <: ExtraIndex : ClassTag](id: ModifierId): Option[T] = + historyStorage.getExtraIndex(id) match { + case Some(m: T) => Some(m) + case _ => None + } + override def contains(id: ModifierId): Boolean = historyStorage.contains(id) /** diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala index eeb624df38..2eae67f973 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -18,7 +18,7 @@ class BalanceInfo(var nanoErgs: Long = 0L, def retreiveAdditionalTokenInfo(history: ErgoHistoryReader): BalanceInfo = { additionalTokenInfo ++= tokens.map(token => { - val iT: IndexedToken = history.typedModifierById[IndexedToken](IndexedTokenSerializer.uniqueId(token._1)).get + val iT: IndexedToken = history.typedExtraIndexById[IndexedToken](IndexedTokenSerializer.uniqueId(token._1)).get (token._1,(iT.name,iT.decimals)) }) this diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala new file mode 100644 index 0000000000..88592af575 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala @@ -0,0 +1,51 @@ +package org.ergoplatform.nodeView.history.extra + +import scorex.core.serialization.ScorexSerializer +import scorex.util.serialization.{Reader, Writer} + +object ExtraIndexSerializer extends ScorexSerializer[ExtraIndex]{ + + override def serialize(obj: ExtraIndex, w: Writer): Unit = { + obj match { + case m: IndexedErgoAddress => + w.put(IndexedErgoAddress.modifierTypeId) + IndexedErgoAddressSerializer.serialize(m, w) + case m: IndexedErgoTransaction => + w.put(IndexedErgoTransaction.modifierTypeId) + IndexedErgoTransactionSerializer.serialize(m, w) + case m: IndexedErgoBox => + w.put(IndexedErgoBox.modifierTypeId) + IndexedErgoBoxSerializer.serialize(m, w) + case m: NumericTxIndex => + w.put(NumericTxIndex.modifierTypeId) + NumericTxIndexSerializer.serialize(m, w) + case m: NumericBoxIndex => + w.put(NumericBoxIndex.modifierTypeId) + NumericBoxIndexSerializer.serialize(m, w) + case m: IndexedToken => + w.put(IndexedToken.modifierTypeId) + IndexedTokenSerializer.serialize(m, w) + case m => + throw new Error(s"Serialization for unknown index: $m") + } + } + + override def parse(r: Reader): ExtraIndex = { + r.getByte() match { + case IndexedErgoAddress.`modifierTypeId` => + IndexedErgoAddressSerializer.parse(r) + case IndexedErgoTransaction.`modifierTypeId` => + IndexedErgoTransactionSerializer.parse(r) + case IndexedErgoBox.`modifierTypeId` => + IndexedErgoBoxSerializer.parse(r) + case NumericTxIndex.`modifierTypeId` => + NumericTxIndexSerializer.parse(r) + case NumericBoxIndex.`modifierTypeId` => + NumericBoxIndexSerializer.parse(r) + case IndexedToken.`modifierTypeId` => + IndexedTokenSerializer.parse(r) + case m => + throw new Error(s"Deserialization for unknown type byte: $m") + } + } + } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 65ec099c89..9a68955f2a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -20,7 +20,11 @@ import java.nio.ByteBuffer import scala.collection.mutable.{ArrayBuffer, ListBuffer} import spire.syntax.all.cfor -class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) +/** + * Actor that constructs an index of database elements. + * @param cacheSettings - cacheSettings to use for saveLimit size + */ +class ExtraIndexer(cacheSettings: CacheSettings) extends Actor with ScorexLogging { private var indexedHeight: Int = 0 @@ -45,13 +49,18 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) private def historyStorage: HistoryStorage = _history.historyStorage // fast access - private val general: ArrayBuffer[BlockSection] = ArrayBuffer.empty[BlockSection] + private val general: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex] private val boxes: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] private val trees: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] // input tokens in a tx private val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] + /** + * Find a box in the boxes buffer. + * @param id - id of the wanted box + * @return an Option containing the index of the wanted box in the boxes buffer or None if box was not found + */ private def findBox(id: BoxId): Option[Int] = { cfor(boxes.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first if (java.util.Arrays.equals(boxes(i).serializedId, id)) @@ -60,14 +69,20 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) None } - // returns index of box in boxes + /** + * Spend an IndexedErgoBox from buffer or database. Also record tokens for later use in balance tracking logic. + * @param id - id of the wanted box + * @param txId - id of the spending transaction + * @param height - height of the block the spending transaction is included in + * @return index of spent box in boxes buffer or -1 if an unknown box was spent + */ private def findAndSpendBox(id: BoxId, txId: ModifierId, height: Int): Int = { findBox(id) match { case Some(i) => tokens ++= boxes(i).asSpent(txId, height).box.additionalTokens.toArray i case None => - history.typedModifierById[IndexedErgoBox](bytesToId(id)) match { // box not found in last saveLimit modifiers + history.typedExtraIndexById[IndexedErgoBox](bytesToId(id)) match { // box not found in last saveLimit modifiers case Some(x) => // box found in DB, update boxes += x.asSpent(txId, height) tokens ++= x.box.additionalTokens.toArray @@ -79,6 +94,11 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } } + /** + * Add or subtract a box from an address in the buffer or in database. + * @param id - hash of the (ergotree) address + * @param spendOrReceive - ErgoBox to spend or IndexedErgoBox to receive + */ private def findAndUpdateTree(id: ModifierId, spendOrReceive: Either[ErgoBox,IndexedErgoBox]): Unit = { cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first if(trees(i).treeHash == id) { // address found in last saveLimit modifiers @@ -89,7 +109,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) return } } - history.typedModifierById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers + history.typedExtraIndexById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers case Some(x) => spendOrReceive match { case Left(box) => trees += x.addTx(globalTxIndex).spendBox(box) // spend box @@ -103,8 +123,14 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } } + /** + * @return number of indexes in all buffers + */ private def modCount: Int = general.length + boxes.length + trees.length + /** + * Write buffered indexes to database and clear buffers. + */ private def saveProgress(): Unit = { val start: Long = System.nanoTime() @@ -116,7 +142,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } // merge all modifiers to an Array, avoids reallocations durin concatenation (++) - val all: Array[BlockSection] = new Array[BlockSection](modCount) + val all: Array[ExtraIndex] = new Array[ExtraIndex](modCount) val offset: Array[Int] = Array(0, general.length, general.length + boxes.length) cfor(0)(_ < general.length, _ + 1) { i => all(i + offset(0)) = general(i) } cfor(0)(_ < boxes.length , _ + 1) { i => all(i + offset(1)) = boxes(i) } @@ -142,6 +168,11 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) lastWroteToDB = indexedHeight } + /** + * Process a batch of BlockTransactions into memory and occasionally write them to database. + * @param bt - BlockTransaction to process + * @param height - height of the block containing the transactions + */ private def index(bt: BlockTransactions, height: Int): Unit = { if(caughtUp && height <= indexedHeight) return // do not process older blocks again after caught up (due to actor message queue) @@ -170,7 +201,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) //process transaction outputs cfor(0)(_ < tx.outputs.size, _ + 1) { i => val box: ErgoBox = tx.outputs(i) - boxes += new IndexedErgoBox(Some(height), None, None, box, globalBoxIndex) // box by id + boxes += new IndexedErgoBox(height, None, None, box, globalBoxIndex) // box by id general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number // box by address @@ -200,7 +231,7 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) indexedHeight = height // update height here after caught up with chain if(modCount >= saveLimit || // modifier limit reached to write to db - history.fullBlockHeight == history.headersHeight) // write to db every block after caught up + history.fullBlockHeight == history.headersHeight) // write to db every block after chain synced saveProgress() }else @@ -209,6 +240,9 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) } + /** + * Main indexer loop that tries to catch up with the already present blocks in database. + */ private def run(): Unit = { indexedHeight = ByteBuffer.wrap(history.modifierBytesById(bytesToId(IndexedHeightKey)) .getOrElse(Array.fill[Byte](4){0})).getInt @@ -245,6 +279,10 @@ class ExtraIndex(chainSettings: ChainSettings, cacheSettings: CacheSettings) object ExtraIndexerRef { object ReceivableMessages { + /** + * Initialize ExtraIndexer and start indexing. + * @param history - handle to database + */ case class Start(history: ErgoHistory) } @@ -265,7 +303,11 @@ object ExtraIndexerRef { index } - // faster id to bytes - no safety checks + /** + * Faster id to bytes - no safety checks + * @param id - ModifierId to convert to byte representation + * @return an array of bytes + */ private[extra] def fastIdToBytes(id: ModifierId): Array[Byte] = { val x: Array[Byte] = new Array[Byte](id.length / 2) cfor(0)(_ < id.length, _ + 2) {i => x(i / 2) = ((hexIndex(id(i)) << 4) | hexIndex(id(i + 1))).toByte} @@ -277,7 +319,7 @@ object ExtraIndexerRef { val GlobalBoxIndexKey: Array[Byte] = Algos.hash("boxes height") def apply(chainSettings: ChainSettings, cacheSettings: CacheSettings)(implicit system: ActorSystem): ActorRef = { - val actor = system.actorOf(Props.create(classOf[ExtraIndex], chainSettings, cacheSettings)) + val actor = system.actorOf(Props.create(classOf[ExtraIndexer], cacheSettings)) _ae = chainSettings.addressEncoder ExtraIndexerRefHolder.init(actor) actor @@ -289,10 +331,22 @@ object ExtraIndexerRefHolder { private var _actor: Option[ActorRef] = None private var running: Boolean = false + /** + * Start stored ExtraIndexer actor with history handle. + * @param history - initialized history handle + */ private[history] def start(history: ErgoHistory): Unit = if(_actor.isDefined && !running) { _actor.get ! Start(history) running = true } + /** + * Store a Ref to an ExtraIndexer actor. + * @param actor - Ref to ExtraIndexer + */ private[extra] def init(actor: ActorRef): Unit = _actor = Some(actor) } +/** + * Base trait for all additional indexes made by ExtraIndexer + */ +trait ExtraIndex extends BlockSection diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 1a160edbe1..3a89683ac2 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -1,7 +1,6 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox -import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getBoxes, getSegmentsForRange, getTxs, segmentTreshold, slice} @@ -16,10 +15,17 @@ import sigmastate.Values.ErgoTree import scala.collection.mutable.ListBuffer import spire.syntax.all.cfor +/** + * An index of an address (ErgoTree) + * @param treeHash - hash of the corresponding ErgoTree + * @param txs - list of numberic transaction indexes associated with this address + * @param boxes - list of numberic box indexes associated with this address + * @param balanceInfo - balance information (Optional because fragments do not contain it) + */ case class IndexedErgoAddress(treeHash: ModifierId, txs: ListBuffer[Long], boxes: ListBuffer[Long], - balanceInfo: Option[BalanceInfo]) extends BlockSection with ScorexLogging { + balanceInfo: Option[BalanceInfo]) extends ExtraIndex with ScorexLogging { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = fastIdToBytes(treeHash) @@ -31,61 +37,109 @@ case class IndexedErgoAddress(treeHash: ModifierId, private[extra] var boxSegmentCount: Int = 0 private[extra] var txSegmentCount: Int = 0 + /** + * @return total number of transactions associated with this address + */ def txCount(): Long = segmentTreshold * txSegmentCount + txs.length + + /** + * @return total number of boxes associated with this address + */ def boxCount(): Long = segmentTreshold * boxSegmentCount + boxes.length + /** + * Get a range of the transactions associated with this address + * @param history - history to use + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @return array of transactions with full bodies + */ def retrieveTxs(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoTransaction] = { if(offset + limit > txs.length && txSegmentCount > 0) { val range: Array[Int] = getSegmentsForRange(offset, limit) val data: ListBuffer[Long] = ListBuffer.empty[Long] cfor(0)(_ < range.length, _ + 1) { i => - history.typedModifierById[IndexedErgoAddress](txSegmentId(treeHash, txSegmentCount - range(i))).get.txs ++=: data + history.typedExtraIndexById[IndexedErgoAddress](txSegmentId(treeHash, txSegmentCount - range(i))).get.txs ++=: data } getTxs(slice(data ++= (if(offset < txs.length) txs else Nil), offset % segmentTreshold, limit))(history) } else getTxs(slice(txs, offset, limit))(history) } + /** + * Get a range of the boxes associated with this address + * @param history - history to use + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @return array of boxes + */ def retrieveBoxes(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoBox] = { if(offset + limit > boxes.length && boxSegmentCount > 0) { val range: Array[Int] = getSegmentsForRange(offset, limit) val data: ListBuffer[Long] = ListBuffer.empty[Long] cfor(0)(_ < range.length, _ + 1) { i => - history.typedModifierById[IndexedErgoAddress](boxSegmentId(treeHash, boxSegmentCount - range(i))).get.boxes ++=: data + history.typedExtraIndexById[IndexedErgoAddress](boxSegmentId(treeHash, boxSegmentCount - range(i))).get.boxes ++=: data } getBoxes(slice(data ++= (if(offset < boxes.length) boxes else Nil), offset % segmentTreshold, limit))(history) } else getBoxes(slice(boxes, offset, limit))(history) } + /** + * Get a range of the boxes associated with this address that are NOT spent + * @param history - history to use + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @return array of unspent boxes + */ def retrieveUtxos(history: ErgoHistoryReader, offset: Int, limit: Int): Array[IndexedErgoBox] = { val data: ListBuffer[IndexedErgoBox] = ListBuffer.empty[IndexedErgoBox] data ++= boxes.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) var segment: Int = boxSegmentCount while(data.length < limit && segment > 0) { segment -= 1 - history.typedModifierById[IndexedErgoAddress](boxSegmentId(treeHash, segment)).get.boxes + history.typedExtraIndexById[IndexedErgoAddress](boxSegmentId(treeHash, segment)).get.boxes .map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filter(!_.trackedBox.isSpent) ++=: data } slice(data, offset, limit).toArray } + /** + * Associate transaction index with this address + * @param tx - numeric transaction index + * @return this address + */ private[extra] def addTx(tx: Long): IndexedErgoAddress = { if(txs.lastOption.getOrElse(-1) != tx) txs += tx // check for duplicates this } + /** + * Associate box with this address and update BalanceInfo + * @param iEb - box to use + * @return this address + */ private[extra] def addBox(iEb: IndexedErgoBox): IndexedErgoAddress = { boxes += iEb.globalIndex balanceInfo.get.add(iEb.box) this } + /** + * Update BalanceInfo by spending a box associated with this address + * @param box - box to spend + * @return this address + */ private[extra] def spendBox(box: ErgoBox): IndexedErgoAddress = { balanceInfo.get.subtract(box) this } + /** + * Create an array addresses each containing a "segmentTreshold" number of this address's transaction and box indexes. + * These special addresses have their ids calculated by "txSegmentId" and "boxSegmentId" respectively. + * @return array of addresses + */ private[extra] def splitToSegments(): Array[IndexedErgoAddress] = { val data: Array[IndexedErgoAddress] = new Array[IndexedErgoAddress]((txs.length / segmentTreshold) + (boxes.length / segmentTreshold)) var i: Int = 0 @@ -109,7 +163,20 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] def hashErgoTree(tree: ErgoTree): Array[Byte] = Algos.hash(tree.bytes) + /** + * Calculates id of an address segment containing box indexes. + * @param hash - hash of the parent addresses ErgoTree + * @param segmentNum - numberic identifier of the segment + * @return calculated ModifierId + */ def boxSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " box segment " + segmentNum)) + + /** + * Calculates id of an address segment transaction indexes. + * @param hash - hash of the parent addresses ErgoTree + * @param segmentNum - numberic identifier of the segment + * @return calculated ModifierId + */ def txSegmentId(hash: ModifierId, segmentNum: Int): ModifierId = bytesToId(Algos.hash(hash + " tx segment " + segmentNum)) override def serialize(iEa: IndexedErgoAddress, w: Writer): Unit = { @@ -145,15 +212,41 @@ object IndexedErgoAddress { val segmentTreshold: Int = 512 + /** + * Calculate the segment offsets for the given range. + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @return array of offsets + */ def getSegmentsForRange(offset: Int, limit: Int): Array[Int] = (math.max(math.ceil(offset * 1F / segmentTreshold).toInt, 1) to math.ceil((offset + limit) * 1F / segmentTreshold).toInt).toArray + /** + * Shorthand to get a range from an array by offset and limit. + * @param arr - array to get range from + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @tparam T - type of "arr" array + * @return range in "arr" array + */ def slice[T](arr: Iterable[T], offset: Int, limit: Int): Iterable[T] = arr.slice(arr.size - offset - limit, arr.size - offset) + /** + * Get an array of transactions with full bodies from an array of numeric transaction indexes + * @param arr - array of numeric transaction indexes to retrieve + * @param history - database handle + * @return array of transactions with full bodies + */ def getTxs(arr: Iterable[Long])(history: ErgoHistoryReader): Array[IndexedErgoTransaction] = // sorted to match explorer arr.map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray.sortBy(tx => (-tx.height, tx.id)) + /** + * Get an array of boxes from an array of numeric box indexes + * @param arr - array of numeric box indexes to retrieve + * @param history - database handle + * @return array of boxes + */ def getBoxes(arr: Iterable[Long])(history: ErgoHistoryReader): Array[IndexedErgoBox] = arr.map(n => NumericBoxIndex.getBoxByNumber(history, n).get).toArray } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala index a41c7ae5e3..ef4f7df8b8 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -1,7 +1,6 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.{ErgoAddress, ErgoBox, Pay2SAddress} -import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import org.ergoplatform.nodeView.wallet.WalletBox import org.ergoplatform.wallet.Constants.ScanId @@ -12,19 +11,27 @@ import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.ErgoTree -class IndexedErgoBox(val inclusionHeightOpt: Option[Int], +/** + * Index of a box. + * @param inclusionHeight - height of the block in which the creating transaction was included in + * @param spendingTxIdOpt - optional, id of the spending transaction + * @param spendingHeightOpt - optional, height of the block in which the spending transaction was included in + * @param box - underlying ErgoBox + * @param globalIndex - numeric index of the box + */ +class IndexedErgoBox(val inclusionHeight: Int, var spendingTxIdOpt: Option[ModifierId], var spendingHeightOpt: Option[Int], val box: ErgoBox, val globalIndex: Long) extends WalletBox(TrackedBox(box.transactionId, box.index, - inclusionHeightOpt, + Some(inclusionHeight), spendingTxIdOpt, spendingHeightOpt, box, Set.empty[ScanId]), - None) with BlockSection { + None) with ExtraIndex { override def parentId: ModifierId = null override val modifierTypeId: ModifierTypeId = IndexedErgoBox.modifierTypeId @@ -33,8 +40,17 @@ class IndexedErgoBox(val inclusionHeightOpt: Option[Int], override type M = IndexedErgoBox override def serializer: ScorexSerializer[IndexedErgoBox] = IndexedErgoBoxSerializer + /** + * @return address constructed from the ErgoTree of this box + */ def getAddress: ErgoAddress = IndexedErgoBoxSerializer.getAddress(box.ergoTree) + /** + * Fill in spending parameters. + * @param txId - id of the spending transaction + * @param txHeight - height of the block in which the spending transaction was included in + * @return this box + */ def asSpent(txId: ModifierId, txHeight: Int): IndexedErgoBox = { spendingTxIdOpt = Some(txId) spendingHeightOpt = Some(txHeight) @@ -50,7 +66,7 @@ object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { } override def serialize(iEb: IndexedErgoBox, w: Writer): Unit = { - w.putOption[Int](iEb.inclusionHeightOpt)(_.putInt(_)) + w.putInt(iEb.inclusionHeight) w.putOption[ModifierId](iEb.spendingTxIdOpt)((ww, id) => ww.putBytes(fastIdToBytes(id))) w.putOption[Int](iEb.spendingHeightOpt)(_.putInt(_)) ErgoBoxSerializer.serialize(iEb.box, w) @@ -58,12 +74,12 @@ object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { } override def parse(r: Reader): IndexedErgoBox = { - val inclusionHeightOpt: Option[Int] = r.getOption[Int](r.getInt()) + val inclusionHeight: Int = r.getInt() val spendingTxIdOpt: Option[ModifierId] = r.getOption[ModifierId](bytesToId(r.getBytes(32))) val spendingHeightOpt: Option[Int] = r.getOption[Int](r.getInt()) val box: ErgoBox = ErgoBoxSerializer.parse(r) val globalIndex: Long = r.getLong() - new IndexedErgoBox(inclusionHeightOpt, spendingTxIdOpt, spendingHeightOpt, box, globalIndex) + new IndexedErgoBox(inclusionHeight, spendingTxIdOpt, spendingHeightOpt, box, globalIndex) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index 6b99bcd10d..f4a9be2e70 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -1,7 +1,6 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.DataInput import org.ergoplatform.modifiers.history.BlockTransactions @@ -11,9 +10,15 @@ import scorex.core.ModifierTypeId import scorex.util.serialization.{Reader, Writer} import scorex.util.{ModifierId, bytesToId} +/** + * Index of a transaction. + * @param txid - id of this transaction + * @param height - height of the block which includes this transaction + * @param globalIndex - numeric index of this transaction + */ case class IndexedErgoTransaction(txid: ModifierId, height: Int, - globalIndex: Long) extends BlockSection { + globalIndex: Long) extends ExtraIndex { override val modifierTypeId: ModifierTypeId = IndexedErgoTransaction.modifierTypeId override def serializedId: Array[Byte] = fastIdToBytes(txid) @@ -42,6 +47,11 @@ case class IndexedErgoTransaction(txid: ModifierId, def outputs: IndexedSeq[IndexedErgoBox] = _outputs def txSize: Int = _txSize + /** + * Get all information related to this transaction from database. + * @param history - database handle + * @return this transaction augmented with additional information + */ def retrieveBody(history: ErgoHistoryReader): IndexedErgoTransaction = { val header: Header = history.typedModifierById[Header](history.bestHeaderIdAtHeight(height).get).get @@ -52,9 +62,9 @@ case class IndexedErgoTransaction(txid: ModifierId, _timestamp = header.timestamp _index = blockTxs.txs.indices.find(blockTxs.txs(_).id == txid).get _numConfirmations = history.bestFullBlockOpt.get.height - height - _inputs = blockTxs.txs(_index).inputs.map(input => history.typedModifierById[IndexedErgoBox](bytesToId(input.boxId)).get) + _inputs = blockTxs.txs(_index).inputs.map(input => history.typedExtraIndexById[IndexedErgoBox](bytesToId(input.boxId)).get) _dataInputs = blockTxs.txs(_index).dataInputs - _outputs = blockTxs.txs(_index).outputs.map(output => history.typedModifierById[IndexedErgoBox](bytesToId(output.id)).get) + _outputs = blockTxs.txs(_index).outputs.map(output => history.typedExtraIndexById[IndexedErgoBox](bytesToId(output.id)).get) _txSize = blockTxs.txs(_index).size this diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 1c727e5370..13211a1bd3 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -2,7 +2,6 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.{R4, R5, R6} -import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.uniqueId import org.ergoplatform.settings.Algos @@ -13,12 +12,21 @@ import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.{CollectionConstant, EvaluatedValue} import sigmastate.{SByte, SType} +/** + * Index of a token containing creation information. + * @param tokenId - id of this token + * @param boxId - id of the creation box + * @param amount - emission amount + * @param name - name of this token (UTF-8) + * @param description - description of this token (UTF-8) + * @param decimals - number of decimal places + */ case class IndexedToken(tokenId: ModifierId, boxId: ModifierId, amount: Long, name: String, description: String, - decimals: Int) extends BlockSection { + decimals: Int) extends ExtraIndex { override def parentId: ModifierId = null override val modifierTypeId: ModifierTypeId = IndexedToken.modifierTypeId override type M = IndexedToken @@ -28,10 +36,19 @@ case class IndexedToken(tokenId: ModifierId, } object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { - - // necessary, because token ids are sometimes identical to box ids, which causes overwrites + /** + * Calculate a unique identifier for this a token. + * Necessary, because token ids are sometimes identical to box ids, which causes overwrites. + * @param tokenId - id of the token + * @return unique id for token + */ def uniqueId(tokenId: ModifierId): ModifierId = bytesToId(Algos.hash(tokenId + "token")) + /** + * Check if a box is creation a token. + * @param box - box to check + * @return true if the box is creation a token, false otherwise + */ def tokenRegistersSet(box: ErgoBox): Boolean = { // registers exist @@ -53,6 +70,12 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { true } + /** + * Get the number of decimals places from a register. + * Try-catch, because some old tokens used Int to store the decimals, rather than Byte Coll + * @param reg - register to extract decimals from + * @return number of decimals places + */ def getDecimals(reg: EvaluatedValue[_ <: SType]): Int = { try { new String(reg.asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8").toInt @@ -61,6 +84,11 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { } } + /** + * Construct a token index from a box. Used after checking box with "tokenRegistersSet". + * @param box - box to use + * @return token index + */ def fromBox(box: ErgoBox): IndexedToken = IndexedToken(bytesToId(box.additionalTokens(0)._1), bytesToId(box.id), diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala index 573b32c9b8..74aa960d51 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala @@ -1,6 +1,5 @@ package org.ergoplatform.nodeView.history.extra -import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import org.ergoplatform.settings.Algos @@ -9,7 +8,12 @@ import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} -case class NumericTxIndex(n: Long, m: ModifierId) extends BlockSection { +/** + * Numeric index pointing to a transaction id. + * @param n - index number of a transaction + * @param m - id of a transaction + */ +case class NumericTxIndex(n: Long, m: ModifierId) extends ExtraIndex { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = NumericTxIndex.indexToBytes(n) override def parentId: ModifierId = null @@ -34,13 +38,29 @@ object NumericTxIndexSerializer extends ScorexSerializer[NumericTxIndex] { object NumericTxIndex { val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 25.toByte + /** + * Convert the index number of a transaction to an id for database retreival. + * @param n - index number to hash + * @return id corresponding to index number + */ def indexToBytes(n: Long): Array[Byte] = Algos.hash("txns height " + n) + /** + * Get a body-less transaction from database by its index number. + * @param history - database handle + * @param n - index number of a transaction + * @return transaction with given index, if found + */ def getTxByNumber(history: ErgoHistoryReader, n: Long): Option[IndexedErgoTransaction] = - history.typedModifierById[IndexedErgoTransaction](history.typedModifierById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(n))).get.m) + history.typedExtraIndexById[IndexedErgoTransaction](history.typedExtraIndexById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(n))).get.m) } -case class NumericBoxIndex(n: Long, m: ModifierId) extends BlockSection { +/** + * Numeric index pointing to a box id. + * @param n - index number of a box + * @param m - id of a box + */ +case class NumericBoxIndex(n: Long, m: ModifierId) extends ExtraIndex { override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = NumericBoxIndex.indexToBytes(n) override def parentId: ModifierId = null @@ -65,8 +85,19 @@ object NumericBoxIndexSerializer extends ScorexSerializer[NumericBoxIndex] { object NumericBoxIndex { val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 30.toByte + /** + * Convert the index number of a box to an id for database retreival. + * @param n - index number to hash + * @return id corresponding to index number + */ def indexToBytes(n: Long): Array[Byte] = Algos.hash("boxes height " + n) + /** + * Get a box from database by its index number. + * @param history - database handle + * @param n - index number of a box + * @return box with given index, if found + */ def getBoxByNumber(history: ErgoHistoryReader, n: Long): Option[IndexedErgoBox] = - history.typedModifierById[IndexedErgoBox](history.typedModifierById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(n))).get.m) + history.typedExtraIndexById[IndexedErgoBox](history.typedExtraIndexById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(n))).get.m) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index b13da291ce..72cef91af7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -4,7 +4,7 @@ import com.github.benmanes.caffeine.cache.Caffeine import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.modifiers.history.HistoryModifierSerializer import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress +import org.ergoplatform.nodeView.history.extra.{ExtraIndexSerializer, ExtraIndex, IndexedErgoAddress} import org.ergoplatform.settings.{Algos, CacheSettings, ErgoSettings} import scorex.core.ModifierTypeId import scorex.core.utils.ScorexEncoding @@ -21,6 +21,7 @@ import spire.syntax.all.cfor * @param indexStore - Additional key-value storage for indexes, required by History for efficient work. * contains links to bestHeader, bestFullBlock, heights and scores for different blocks, etc. * @param objectsStore - key-value store, where key is id of ErgoPersistentModifier and value is it's bytes + * @param extraStore - key-value store, where key is id of Index and value is it's bytes * @param config - cache configs */ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStore: LDBKVStore, config: CacheSettings) @@ -41,7 +42,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e private val extraCache = Caffeine.newBuilder() .maximumSize(config.history.extraCacheSize) - .build[String, BlockSection]() + .build[String, ExtraIndex]() private val indexCache = Caffeine.newBuilder() @@ -50,12 +51,12 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e private def cacheModifier(mod: BlockSection): Unit = mod.modifierTypeId match { case Header.modifierTypeId => headersCache.put(mod.id, mod) - case IndexedErgoAddress.modifierTypeId => extraCache.put(mod.id, mod) // only cache "big" modifiers + case IndexedErgoAddress.modifierTypeId => extraCache.put(mod.id, mod.asInstanceOf[ExtraIndex]) // only cache "big" modifiers case _ => blockSectionsCache.put(mod.id, mod) } private def lookupModifier(id: ModifierId): Option[BlockSection] = - Option(extraCache.getIfPresent(id)) orElse Option(headersCache.getIfPresent(id)) orElse Option(blockSectionsCache.getIfPresent(id)) + Option(headersCache.getIfPresent(id)) orElse Option(blockSectionsCache.getIfPresent(id)) private def removeModifier(id: ModifierId): Unit = { headersCache.invalidate(id) @@ -72,18 +73,31 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e } def modifierById(id: ModifierId): Option[BlockSection] = - lookupModifier(id) orElse - (extraStore.get(idToBytes(id)) orElse objectsStore.get(idToBytes(id))).flatMap { bytes => - HistoryModifierSerializer.parseBytesTry(bytes) match { - case Success(pm) => - log.trace(s"Cache miss for existing modifier $id") - cacheModifier(pm) - Some(pm) - case Failure(_) => - log.warn(s"Failed to parse modifier ${encoder.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") - None - } + lookupModifier(id) orElse objectsStore.get(idToBytes(id)).flatMap { bytes => + HistoryModifierSerializer.parseBytesTry(bytes) match { + case Success(pm) => + log.trace(s"Cache miss for existing modifier $id") + cacheModifier(pm) + Some(pm) + case Failure(_) => + log.warn(s"Failed to parse modifier ${encoder.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") + None } + } + + def getExtraIndex(id: ModifierId): Option[ExtraIndex] = { + Option(extraCache.getIfPresent(id)) orElse extraStore.get(idToBytes(id)).flatMap { bytes => + ExtraIndexSerializer.parseBytesTry(bytes) match { + case Success(pm) => + log.trace(s"Cache miss for existing index $id") + cacheModifier(pm) + Some(pm) + case Failure(_) => + log.warn(s"Failed to parse index ${encoder.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") + None + } + } + } def getIndex(id: ByteArrayWrapper): Option[Array[Byte]] = Option(indexCache.getIfPresent(id)).orElse { @@ -113,8 +127,8 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e } def insertExtra(indexesToInsert: Array[(Array[Byte], Array[Byte])], - objectsToInsert: Array[BlockSection]): Unit = { - extraStore.insert(objectsToInsert.map(mod => (mod.serializedId, HistoryModifierSerializer.toBytes(mod)))) + objectsToInsert: Array[ExtraIndex]): Unit = { + extraStore.insert(objectsToInsert.map(mod => (mod.serializedId, ExtraIndexSerializer.toBytes(mod)))) cfor(0)(_ < objectsToInsert.length, _ + 1) { i => cacheModifier(objectsToInsert(i))} cfor(0)(_ < indexesToInsert.length, _ + 1) { i => extraStore.insert(indexesToInsert(i)._1, indexesToInsert(i)._2)} } From 3996b5c2b0d420328035429e088f69939ed2fc2c Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Mon, 24 Oct 2022 20:06:13 +0300 Subject: [PATCH 41/90] ExtraIndex not inherited from BlockSection --- .../nodeView/history/extra/ExtraIndexer.scala | 6 ++++-- .../nodeView/history/extra/IndexedErgoAddress.scala | 5 ----- .../nodeView/history/extra/IndexedErgoBox.scala | 5 ----- .../history/extra/IndexedErgoTransaction.scala | 5 ----- .../nodeView/history/extra/IndexedToken.scala | 7 ++----- .../nodeView/history/extra/NumericIndex.scala | 12 ++---------- .../nodeView/history/storage/HistoryStorage.scala | 7 ++++--- 7 files changed, 12 insertions(+), 35 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 9a68955f2a..6190b12bf7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -2,7 +2,6 @@ package org.ergoplatform.nodeView.history.extra import akka.actor.{Actor, ActorRef, ActorSystem, Props} import org.ergoplatform.ErgoBox.{BoxId, TokenId} -import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.modifiers.history.header.Header @@ -349,4 +348,7 @@ object ExtraIndexerRefHolder { /** * Base trait for all additional indexes made by ExtraIndexer */ -trait ExtraIndex extends BlockSection +trait ExtraIndex { + def id: ModifierId = bytesToId(serializedId) + def serializedId: Array[Byte] +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 3a89683ac2..9b12c5dbef 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -27,12 +27,7 @@ case class IndexedErgoAddress(treeHash: ModifierId, boxes: ListBuffer[Long], balanceInfo: Option[BalanceInfo]) extends ExtraIndex with ScorexLogging { - override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = fastIdToBytes(treeHash) - override def parentId: ModifierId = null - override val modifierTypeId: ModifierTypeId = IndexedErgoAddress.modifierTypeId - override type M = IndexedErgoAddress - override def serializer: ScorexSerializer[IndexedErgoAddress] = IndexedErgoAddressSerializer private[extra] var boxSegmentCount: Int = 0 private[extra] var txSegmentCount: Int = 0 diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala index ef4f7df8b8..f6087d2c31 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -33,12 +33,7 @@ class IndexedErgoBox(val inclusionHeight: Int, Set.empty[ScanId]), None) with ExtraIndex { - override def parentId: ModifierId = null - override val modifierTypeId: ModifierTypeId = IndexedErgoBox.modifierTypeId - override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = box.id - override type M = IndexedErgoBox - override def serializer: ScorexSerializer[IndexedErgoBox] = IndexedErgoBoxSerializer /** * @return address constructed from the ErgoTree of this box diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index f4a9be2e70..f1fd68f456 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -20,12 +20,7 @@ case class IndexedErgoTransaction(txid: ModifierId, height: Int, globalIndex: Long) extends ExtraIndex { - override val modifierTypeId: ModifierTypeId = IndexedErgoTransaction.modifierTypeId override def serializedId: Array[Byte] = fastIdToBytes(txid) - override val sizeOpt: Option[Int] = None - override def parentId: ModifierId = null - override type M = IndexedErgoTransaction - override def serializer: ScorexSerializer[IndexedErgoTransaction] = IndexedErgoTransactionSerializer private var _blockId: ModifierId = ModifierId @@ "" private var _inclusionHeight: Int = 0 diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 13211a1bd3..6f27269d7b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -27,12 +27,9 @@ case class IndexedToken(tokenId: ModifierId, name: String, description: String, decimals: Int) extends ExtraIndex { - override def parentId: ModifierId = null - override val modifierTypeId: ModifierTypeId = IndexedToken.modifierTypeId - override type M = IndexedToken - override def serializer: ScorexSerializer[IndexedToken] = IndexedTokenSerializer - override val sizeOpt: Option[Int] = None + override def serializedId: Array[Byte] = fastIdToBytes(uniqueId(tokenId)) + } object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala index 74aa960d51..4c0852d0a5 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala @@ -14,13 +14,9 @@ import scorex.util.serialization.{Reader, Writer} * @param m - id of a transaction */ case class NumericTxIndex(n: Long, m: ModifierId) extends ExtraIndex { - override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = NumericTxIndex.indexToBytes(n) - override def parentId: ModifierId = null - override val modifierTypeId: ModifierTypeId = NumericTxIndex.modifierTypeId - override type M = NumericTxIndex - override def serializer: ScorexSerializer[NumericTxIndex] = NumericTxIndexSerializer } + object NumericTxIndexSerializer extends ScorexSerializer[NumericTxIndex] { override def serialize(ni: NumericTxIndex, w: Writer): Unit = { @@ -61,13 +57,9 @@ object NumericTxIndex { * @param m - id of a box */ case class NumericBoxIndex(n: Long, m: ModifierId) extends ExtraIndex { - override val sizeOpt: Option[Int] = None override def serializedId: Array[Byte] = NumericBoxIndex.indexToBytes(n) - override def parentId: ModifierId = null - override val modifierTypeId: ModifierTypeId = NumericBoxIndex.modifierTypeId - override type M = NumericBoxIndex - override def serializer: ScorexSerializer[NumericBoxIndex] = NumericBoxIndexSerializer } + object NumericBoxIndexSerializer extends ScorexSerializer[NumericBoxIndex] { override def serialize(ni: NumericBoxIndex, w: Writer): Unit = { diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 72cef91af7..8eb92617ea 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -51,7 +51,6 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e private def cacheModifier(mod: BlockSection): Unit = mod.modifierTypeId match { case Header.modifierTypeId => headersCache.put(mod.id, mod) - case IndexedErgoAddress.modifierTypeId => extraCache.put(mod.id, mod.asInstanceOf[ExtraIndex]) // only cache "big" modifiers case _ => blockSectionsCache.put(mod.id, mod) } @@ -90,7 +89,9 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e ExtraIndexSerializer.parseBytesTry(bytes) match { case Success(pm) => log.trace(s"Cache miss for existing index $id") - cacheModifier(pm) + if(pm.isInstanceOf[IndexedErgoAddress]){ + extraCache.put(pm.id, pm) // only cache addresses + } Some(pm) case Failure(_) => log.warn(s"Failed to parse index ${encoder.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") @@ -129,7 +130,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e def insertExtra(indexesToInsert: Array[(Array[Byte], Array[Byte])], objectsToInsert: Array[ExtraIndex]): Unit = { extraStore.insert(objectsToInsert.map(mod => (mod.serializedId, ExtraIndexSerializer.toBytes(mod)))) - cfor(0)(_ < objectsToInsert.length, _ + 1) { i => cacheModifier(objectsToInsert(i))} + cfor(0)(_ < objectsToInsert.length, _ + 1) { i => val ei = objectsToInsert(i); extraCache.put(ei.id, ei)} cfor(0)(_ < indexesToInsert.length, _ + 1) { i => extraStore.insert(indexesToInsert(i)._1, indexesToInsert(i)._2)} } From f389646b912fb25906d8e813d9b4ffe6d2cd2300 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Mon, 24 Oct 2022 21:44:50 +0200 Subject: [PATCH 42/90] minor changes --- .../org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala | 2 +- .../nodeView/history/extra/IndexedErgoAddress.scala | 1 + .../nodeView/history/extra/IndexedErgoTransaction.scala | 1 + .../org/ergoplatform/nodeView/history/extra/IndexedToken.scala | 3 ++- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 6190b12bf7..1c7dfed627 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -100,7 +100,7 @@ class ExtraIndexer(cacheSettings: CacheSettings) */ private def findAndUpdateTree(id: ModifierId, spendOrReceive: Either[ErgoBox,IndexedErgoBox]): Unit = { cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(trees(i).treeHash == id) { // address found in last saveLimit modifiers + if(trees(i).id == id) { // address found in last saveLimit modifiers spendOrReceive match { case Left(box) => trees(i).addTx(globalTxIndex).spendBox(box) // spend box case Right(iEb) => trees(i).addTx(globalTxIndex).addBox(iEb) // receive box diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 9b12c5dbef..165b6b99db 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -27,6 +27,7 @@ case class IndexedErgoAddress(treeHash: ModifierId, boxes: ListBuffer[Long], balanceInfo: Option[BalanceInfo]) extends ExtraIndex with ScorexLogging { + override def id: ModifierId = treeHash override def serializedId: Array[Byte] = fastIdToBytes(treeHash) private[extra] var boxSegmentCount: Int = 0 diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index f1fd68f456..3e1d489900 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -20,6 +20,7 @@ case class IndexedErgoTransaction(txid: ModifierId, height: Int, globalIndex: Long) extends ExtraIndex { + override def id: ModifierId = txid override def serializedId: Array[Byte] = fastIdToBytes(txid) private var _blockId: ModifierId = ModifierId @@ "" diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 6f27269d7b..31582583a1 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -28,6 +28,7 @@ case class IndexedToken(tokenId: ModifierId, description: String, decimals: Int) extends ExtraIndex { + override def id: ModifierId = uniqueId(tokenId) override def serializedId: Array[Byte] = fastIdToBytes(uniqueId(tokenId)) } @@ -42,7 +43,7 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { def uniqueId(tokenId: ModifierId): ModifierId = bytesToId(Algos.hash(tokenId + "token")) /** - * Check if a box is creation a token. + * Check if a box is creating a token. * @param box - box to check * @return true if the box is creation a token, false otherwise */ From 9e205027f662e6f5a369dc6197c95e91d4600c87 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Tue, 22 Nov 2022 01:08:21 +0300 Subject: [PATCH 43/90] CrawlerRunner removed, Rollback signal sent --- .../ergoplatform/bench/CrawlerRunner.scala | 76 ------------------- .../network/ErgoNodeViewSynchronizer.scala | 2 +- .../nodeView/ErgoNodeViewHolder.scala | 4 + 3 files changed, 5 insertions(+), 77 deletions(-) delete mode 100644 benchmarks/src/test/scala/org/ergoplatform/bench/CrawlerRunner.scala diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/CrawlerRunner.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/CrawlerRunner.scala deleted file mode 100644 index 1be231ebd1..0000000000 --- a/benchmarks/src/test/scala/org/ergoplatform/bench/CrawlerRunner.scala +++ /dev/null @@ -1,76 +0,0 @@ -package org.ergoplatform.bench - -import java.io.File - -import akka.actor.ActorRef -import org.ergoplatform.bench.misc.{CrawlerConfig, TempDir} -import org.ergoplatform.http.api.{BlocksApiRoute, ErgoUtilsApiRoute, InfoApiRoute, TransactionsApiRoute} -import org.ergoplatform.local.ErgoStatsCollectorRef -import org.ergoplatform.mining.ErgoMiner -import org.ergoplatform.mining.emission.EmissionRules -import org.ergoplatform.network.{ErgoNodeViewSynchronizer, ErgoSyncTracker} -import org.ergoplatform.nodeView.history.ErgoSyncInfoMessageSpec -import org.ergoplatform.nodeView.{ErgoNodeViewRef, ErgoReadersHolderRef} -import org.ergoplatform.settings.{Args, ErgoSettings} -import scorex.core.api.http.ApiRoute -import scorex.core.app.Application -import scorex.core.network.{DeliveryTracker, PeerFeature} -import scorex.core.network.message.MessageSpec -import scorex.core.settings.ScorexSettings - -import scala.concurrent.ExecutionContextExecutor -import scala.io.Source - -class CrawlerRunner(args: Array[String]) extends Application { - - lazy val fileToSave: String = args.headOption.getOrElse("/") - lazy val threshold: Int = args.lift(1).getOrElse("15000").toInt - lazy val cfgPath: Option[String] = args.lift(2) - lazy val benchConfig: CrawlerConfig = CrawlerConfig(fileToSave, threshold) - lazy val tempDir: File = TempDir.createTempDir - - log.info(s"Temp dir is ${tempDir.getAbsolutePath}") - log.info(s"Config is $benchConfig") - - override protected lazy val features: Seq[PeerFeature] = Seq() - - implicit val ec: ExecutionContextExecutor = actorSystem.dispatcher - - override val ergoSettings: ErgoSettings = ErgoSettings.read(Args(cfgPath, None)) - - lazy val emission = new EmissionRules(ergoSettings.chainSettings.monetary) - - override implicit lazy val scorexSettings: ScorexSettings = ergoSettings.scorexSettings - - override protected lazy val additionalMessageSpecs: Seq[MessageSpec[_]] = Seq(ErgoSyncInfoMessageSpec) - override val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) - - val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) - - val minerRef: ActorRef = ErgoMiner(ergoSettings, nodeViewHolderRef, readersHolderRef, timeProvider) - - private val syncTracker = ErgoSyncTracker(scorexSettings.network, timeProvider) - - val statsCollectorRef: ActorRef = - ErgoStatsCollectorRef(nodeViewHolderRef, networkControllerRef, syncTracker, ergoSettings, timeProvider) - - - override val apiRoutes: Seq[ApiRoute] = Seq( - ErgoUtilsApiRoute(ergoSettings), - InfoApiRoute(statsCollectorRef, scorexSettings.restApi, timeProvider), - BlocksApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings), - TransactionsApiRoute(readersHolderRef, nodeViewHolderRef, ergoSettings)) - - override val swaggerConfig: String = Source.fromResource("api/openapi.yaml").getLines.mkString("\n") - - private val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(ergoSettings) - - override val nodeViewSynchronizer: ActorRef = - ErgoNodeViewSynchronizer(networkControllerRef, nodeViewHolderRef, ErgoSyncInfoMessageSpec, - ergoSettings, timeProvider, syncTracker, deliveryTracker) - -} - -object CrawlerRunner { - def main(args: Array[String]): Unit = new CrawlerRunner(args).run() -} diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 6d5d25e2ce..8ba105d827 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -1216,7 +1216,7 @@ object ErgoNodeViewSynchronizer { case class ChangedState(reader: ErgoStateReader) extends NodeViewChange - //todo: consider sending info on the rollback + case class Rollback(branchPoint: ModifierId) extends NodeViewHolderEvent case object RollbackFailed extends NodeViewHolderEvent diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index fdf52d4693..0837facdaf 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -495,6 +495,10 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti updateNodeView(Some(newHistory), Some(newMinState), Some(newVault), Some(newMemPool)) chainProgress = Some(ChainProgress(pmod, headersHeight, fullBlockHeight, timeProvider.time())) + + if (progressInfo.chainSwitchingNeeded) { + context.system.eventStream.publish(Rollback(progressInfo.branchPoint.get)) + } case Failure(e) => log.warn(s"Can`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod) to minimal state", e) updateNodeView(updatedHistory = Some(newHistory)) From 3553ae122913f9f5a92aaf271a388d21fa5c0cf8 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Wed, 23 Nov 2022 10:10:20 +0100 Subject: [PATCH 44/90] rollback half done --- .../nodeView/history/extra/ExtraIndexer.scala | 51 ++++++++++++++++--- .../history/storage/HistoryStorage.scala | 6 +++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 1c7dfed627..aefc444837 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -6,11 +6,12 @@ import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.ErgoTransaction -import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.FullBlockApplied +import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.{FullBlockApplied, Rollback} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{GlobalBoxIndexKey, GlobalTxIndexKey, IndexedHeightKey} import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.segmentTreshold +import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.hashErgoTree import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.settings.{Algos, CacheSettings, ChainSettings} import scorex.util.{ModifierId, ScorexLogging, bytesToId} @@ -40,6 +41,7 @@ class ExtraIndexer(cacheSettings: CacheSettings) private val saveLimit: Int = cacheSettings.history.extraCacheSize * 10 private var caughtUp: Boolean = false + private var rollback: Boolean = false private def chainHeight: Int = _history.fullBlockHeight @@ -193,7 +195,7 @@ class ExtraIndexer(cacheSettings: CacheSettings) if(height != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => val boxIndex: Int = findAndSpendBox(tx.inputs(i).boxId, tx.id, height) - if(boxIndex >= 0) findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(boxes(boxIndex).box.ergoTree)), Left(boxes(boxIndex).box)) // spend box and add tx + if(boxIndex >= 0) findAndUpdateTree(bytesToId(hashErgoTree(boxes(boxIndex).box.ergoTree)), Left(boxes(boxIndex).box)) // spend box and add tx } } @@ -204,7 +206,7 @@ class ExtraIndexer(cacheSettings: CacheSettings) general += NumericBoxIndex(globalBoxIndex, bytesToId(box.id)) // box id by global box number // box by address - findAndUpdateTree(bytesToId(IndexedErgoAddressSerializer.hashErgoTree(box.ergoTree)), Right(boxes(findBox(box.id).get))) + findAndUpdateTree(bytesToId(hashErgoTree(box.ergoTree)), Right(boxes(findBox(box.id).get))) // check if box is creating a new token, if yes record it if(box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) @@ -250,18 +252,49 @@ class ExtraIndexer(cacheSettings: CacheSettings) log.info(s"Started extra indexer at height $indexedHeight") - while(indexedHeight < chainHeight) { + while(indexedHeight < chainHeight && !rollback) { indexedHeight += 1 index(history.bestBlockTransactionsAt(indexedHeight).get, indexedHeight) } - caughtUp = true + if(rollback) + caughtUp = true + log.info("Indexer caught up with chain") + } + + /** + * Remove all indexes after given height. + * @param height - starting height + */ + def removeAfter(height: Int): Unit = { + + log.info(s"Performing rollback from $indexedHeight to $height") - log.info("Indexer caught up with chain") + cfor(indexedHeight)(_ > height, _ - 1) { block => // remove all block parts down to "height" + val bt: Array[ErgoTransaction] = history.bestBlockTransactionsAt(block).get.txs.toArray + cfor(bt.length - 1)(_ >= 0, _ - 1) { tx => + val boxes: Array[ErgoBox] = bt(tx).outputs.toArray + cfor(boxes.length - 1)(_ >= 0, _ - 1) { box => // revert balances + val address: IndexedErgoAddress = history.typedExtraIndexById[IndexedErgoAddress](bytesToId(hashErgoTree(boxes(box).ergoTree))).get + // TODO + } + historyStorage.removeExtra(boxes.map(x => bytesToId(x.id))) // remove all boxes in tx + } + historyStorage.removeExtra(bt.map(_.id)) // remove all txs in block + } + + // save new height and start indexer + indexedHeight = height + indexedHeightBuffer.clear() + historyStorage.insertExtra(Array((IndexedHeightKey, indexedHeightBuffer.putInt(indexedHeight).array)), Array.empty) + run() + rollback = false } - override def preStart(): Unit = + override def preStart(): Unit = { context.system.eventStream.subscribe(self, classOf[FullBlockApplied]) + context.system.eventStream.subscribe(self, classOf[Rollback]) + } override def postStop(): Unit = log.info(s"Stopped extra indexer at height ${if(lastWroteToDB > 0) lastWroteToDB else indexedHeight}") @@ -269,6 +302,10 @@ class ExtraIndexer(cacheSettings: CacheSettings) override def receive: Receive = { case FullBlockApplied(header: Header) if caughtUp => index(history.typedModifierById[BlockTransactions](header.transactionsId).get, header.height) // after the indexer caught up with the chain, stay up to date + case Rollback(branchPoint: ModifierId) => + val branchHeight: Int = history.heightOf(branchPoint).get + rollback = branchHeight < indexedHeight + if(rollback) removeAfter(branchHeight) case Start(history: ErgoHistory) => _history = history run() diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 8eb92617ea..49d7a745f5 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -134,6 +134,11 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e cfor(0)(_ < indexesToInsert.length, _ + 1) { i => extraStore.insert(indexesToInsert(i)._1, indexesToInsert(i)._2)} } + def removeExtra(indexesToRemove: Array[ModifierId]) : Unit = { + extraStore.remove(indexesToRemove.map(idToBytes)) + cfor(0)(_ < indexesToRemove.length, _ + 1) { i => removeModifier(indexesToRemove(i)) } + } + /** * Insert single object to database. This version allows for efficient insert * when identifier and bytes of object (i.e. modifier, a block section) are known. @@ -168,6 +173,7 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e override def close(): Unit = { log.warn("Closing history storage...") + extraStore.close() indexStore.close() objectsStore.close() } From 6ef6241e30f9f45fbe4e0c96ec7d0fb607902a6b Mon Sep 17 00:00:00 2001 From: jellymlg Date: Thu, 24 Nov 2022 02:23:26 +0100 Subject: [PATCH 45/90] [UNTESTED] finished rollback --- .../nodeView/history/extra/BalanceInfo.scala | 6 +- .../nodeView/history/extra/ExtraIndexer.scala | 66 ++++++++++++++----- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala index 2eae67f973..abd9c7ff63 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -3,6 +3,8 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes +import org.ergoplatform.nodeView.history.extra.IndexedErgoBoxSerializer.getAddress +import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.uniqueId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, ScorexLogging, bytesToId} import scorex.util.serialization.{Reader, Writer} @@ -18,7 +20,7 @@ class BalanceInfo(var nanoErgs: Long = 0L, def retreiveAdditionalTokenInfo(history: ErgoHistoryReader): BalanceInfo = { additionalTokenInfo ++= tokens.map(token => { - val iT: IndexedToken = history.typedExtraIndexById[IndexedToken](IndexedTokenSerializer.uniqueId(token._1)).get + val iT: IndexedToken = history.typedExtraIndexById[IndexedToken](uniqueId(token._1)).get (token._1,(iT.name,iT.decimals)) }) this @@ -53,7 +55,7 @@ class BalanceInfo(var nanoErgs: Long = 0L, tokens.remove(n) else tokens(n) = (id, newVal) - case None => log.warn(s"Failed to subtract token $id from address ${IndexedErgoBoxSerializer.getAddress(box.ergoTree)}") + case None => log.warn(s"Failed to subtract token $id from address ${getAddress(box.ergoTree)}") } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index aefc444837..86d5facd9d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -99,15 +99,16 @@ class ExtraIndexer(cacheSettings: CacheSettings) * Add or subtract a box from an address in the buffer or in database. * @param id - hash of the (ergotree) address * @param spendOrReceive - ErgoBox to spend or IndexedErgoBox to receive + * @return index of updated tree in buffer or -1 if the tree was unknown */ - private def findAndUpdateTree(id: ModifierId, spendOrReceive: Either[ErgoBox,IndexedErgoBox]): Unit = { + private def findAndUpdateTree(id: ModifierId, spendOrReceive: Either[ErgoBox,IndexedErgoBox]): Int = { cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first if(trees(i).id == id) { // address found in last saveLimit modifiers spendOrReceive match { case Left(box) => trees(i).addTx(globalTxIndex).spendBox(box) // spend box case Right(iEb) => trees(i).addTx(globalTxIndex).addBox(iEb) // receive box } - return + return i } } history.typedExtraIndexById[IndexedErgoAddress](id) match { // address not found in last saveLimit modifiers @@ -116,11 +117,13 @@ class ExtraIndexer(cacheSettings: CacheSettings) case Left(box) => trees += x.addTx(globalTxIndex).spendBox(box) // spend box case Right(iEb) => trees += x.addTx(globalTxIndex).addBox(iEb) // receive box } + trees.length - 1 case None => // address not found at all spendOrReceive match { case Left(box) => log.warn(s"Unknown address spent box ${bytesToId(box.id)}") // spend box should never happen by an unknown address case Right(iEb) => trees += IndexedErgoAddress(id, ListBuffer(globalTxIndex), ListBuffer.empty[Long], Some(new BalanceInfo)).addBox(iEb) // receive box } + -1 } } @@ -132,7 +135,7 @@ class ExtraIndexer(cacheSettings: CacheSettings) /** * Write buffered indexes to database and clear buffers. */ - private def saveProgress(): Unit = { + private def saveProgress(writeLog: Boolean = true): Unit = { val start: Long = System.nanoTime() @@ -159,7 +162,8 @@ class ExtraIndexer(cacheSettings: CacheSettings) val end: Long = System.nanoTime() - log.info(s"Processed ${trees.length} ErgoTrees with ${boxes.length} boxes and inserted them to database in ${(end - start) / 1000000D}ms") + if(writeLog) + log.info(s"Processed ${trees.length} ErgoTrees with ${boxes.length} boxes and inserted them to database in ${(end - start) / 1000000D}ms") // clear buffers for next batch general.clear() @@ -268,27 +272,55 @@ class ExtraIndexer(cacheSettings: CacheSettings) */ def removeAfter(height: Int): Unit = { - log.info(s"Performing rollback from $indexedHeight to $height") + saveProgress(false) + + log.info(s"Rolling back indexes from $indexedHeight to $height") cfor(indexedHeight)(_ > height, _ - 1) { block => // remove all block parts down to "height" + val bt: Array[ErgoTransaction] = history.bestBlockTransactionsAt(block).get.txs.toArray cfor(bt.length - 1)(_ >= 0, _ - 1) { tx => - val boxes: Array[ErgoBox] = bt(tx).outputs.toArray - cfor(boxes.length - 1)(_ >= 0, _ - 1) { box => // revert balances - val address: IndexedErgoAddress = history.typedExtraIndexById[IndexedErgoAddress](bytesToId(hashErgoTree(boxes(box).ergoTree))).get - // TODO + + val outputs: Array[ErgoBox] = bt(tx).outputs.toArray + cfor(outputs.length - 1)(_ >= 0, _ - 1) { n => // revert outputs from addresses + + val i: Int = findAndUpdateTree(bytesToId(hashErgoTree(outputs(n).ergoTree)), Left(outputs(n))) // remove this boxes ergs/tokens + trees(i).txs.remove(trees(i).txs.length - 1) // remove tx number duplicate + trees(i).boxes.remove(trees(i).boxes.length - 1) // remove box number + + } + + boxes ++= bt(tx).inputs.map(x => history.typedExtraIndexById[IndexedErgoBox](bytesToId(x.boxId)).get) + cfor(boxes.length - 1)(_ >= 0, _ - 1) { n => // revert inputs from addresses + + boxes(n).spendingTxIdOpt = None + boxes(n).spendingHeightOpt = None + val i: Int = findAndUpdateTree(bytesToId(hashErgoTree(boxes(n).box.ergoTree)), Right(boxes(n))) // add this boxes ergs/tokens + trees(i).txs.remove(trees(i).txs.length - 1) // remove tx number duplicate + trees(i).boxes.remove(trees(i).boxes.length - 1) // remove box number duplicate + } - historyStorage.removeExtra(boxes.map(x => bytesToId(x.id))) // remove all boxes in tx + + historyStorage.removeExtra(outputs.map(x => bytesToId(x.id))) // remove all boxes in tx + } + + // save updated height and indexes + + indexedHeight = block + globalTxIndex = history.typedExtraIndexById[IndexedErgoTransaction](bt.last.id).get.globalIndex + globalBoxIndex = history.typedExtraIndexById[IndexedErgoBox](bytesToId(bt.last.outputs.last.id)).get.globalIndex + saveProgress(false) + historyStorage.removeExtra(bt.map(_.id)) // remove all txs in block } - // save new height and start indexer - indexedHeight = height - indexedHeightBuffer.clear() - historyStorage.insertExtra(Array((IndexedHeightKey, indexedHeightBuffer.putInt(indexedHeight).array)), Array.empty) - run() + log.info(s"Successfully rolled back indexes to $height") + + // restart indexer rollback = false + run() + } override def preStart(): Unit = { @@ -300,15 +332,19 @@ class ExtraIndexer(cacheSettings: CacheSettings) log.info(s"Stopped extra indexer at height ${if(lastWroteToDB > 0) lastWroteToDB else indexedHeight}") override def receive: Receive = { + case FullBlockApplied(header: Header) if caughtUp => index(history.typedModifierById[BlockTransactions](header.transactionsId).get, header.height) // after the indexer caught up with the chain, stay up to date + case Rollback(branchPoint: ModifierId) => val branchHeight: Int = history.heightOf(branchPoint).get rollback = branchHeight < indexedHeight if(rollback) removeAfter(branchHeight) + case Start(history: ErgoHistory) => _history = history run() + } } From 0885936a5350ac7460a8817784e6636a297c33ab Mon Sep 17 00:00:00 2001 From: jellymlg Date: Fri, 23 Dec 2022 14:56:36 +0100 Subject: [PATCH 46/90] refactored ExtraIndexer to use trait and started working on test --- .../nodeView/history/extra/ExtraIndexer.scala | 115 +++++++++++------- .../extra/ExtraIndexerSpecification.scala | 28 +++++ 2 files changed, 97 insertions(+), 46 deletions(-) create mode 100644 src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 86d5facd9d..5dd4ee1af4 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -21,44 +21,52 @@ import scala.collection.mutable.{ArrayBuffer, ListBuffer} import spire.syntax.all.cfor /** - * Actor that constructs an index of database elements. - * @param cacheSettings - cacheSettings to use for saveLimit size + * Base trait for extra indexer actor and its test. */ -class ExtraIndexer(cacheSettings: CacheSettings) - extends Actor with ScorexLogging { +trait ExtraIndexerBase extends ScorexLogging { - private var indexedHeight: Int = 0 - private val indexedHeightBuffer : ByteBuffer = ByteBuffer.allocate(4) + // Indexed block height + protected var indexedHeight: Int = 0 + private val indexedHeightBuffer: ByteBuffer = ByteBuffer.allocate(4) - private var globalTxIndex: Long = 0L + // Indexed transaction count + protected var globalTxIndex: Long = 0L private val globalTxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) - private var globalBoxIndex: Long = 0L + // Indexed box count + protected var globalBoxIndex: Long = 0L private val globalBoxIndexBuffer: ByteBuffer = ByteBuffer.allocate(8) - private var lastWroteToDB: Int = 0 + // Last block height when buffer contents were saved to database + protected var lastWroteToDB: Int = 0 + + // Max buffer size (determined by config) + protected val saveLimit: Int - private val saveLimit: Int = cacheSettings.history.extraCacheSize * 10 + // Flag to signal when indexer has reached current block height + protected var caughtUp: Boolean = false - private var caughtUp: Boolean = false - private var rollback: Boolean = false + // Flag to signal a rollback + protected var rollback: Boolean = false - private def chainHeight: Int = _history.fullBlockHeight + // Database handle + protected var _history: ErgoHistory = null - private var _history: ErgoHistory = null - private def history: ErgoHistoryReader = _history.getReader - private def historyStorage: HistoryStorage = _history.historyStorage + protected def chainHeight: Int = _history.fullBlockHeight + protected def history: ErgoHistoryReader = _history.getReader + protected def historyStorage: HistoryStorage = _history.historyStorage - // fast access + // fast access buffers private val general: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex] private val boxes: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] private val trees: ArrayBuffer[IndexedErgoAddress] = ArrayBuffer.empty[IndexedErgoAddress] // input tokens in a tx - private val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] + protected val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] /** * Find a box in the boxes buffer. + * * @param id - id of the wanted box * @return an Option containing the index of the wanted box in the boxes buffer or None if box was not found */ @@ -72,6 +80,7 @@ class ExtraIndexer(cacheSettings: CacheSettings) /** * Spend an IndexedErgoBox from buffer or database. Also record tokens for later use in balance tracking logic. + * * @param id - id of the wanted box * @param txId - id of the spending transaction * @param height - height of the block the spending transaction is included in @@ -97,13 +106,14 @@ class ExtraIndexer(cacheSettings: CacheSettings) /** * Add or subtract a box from an address in the buffer or in database. + * * @param id - hash of the (ergotree) address * @param spendOrReceive - ErgoBox to spend or IndexedErgoBox to receive * @return index of updated tree in buffer or -1 if the tree was unknown */ - private def findAndUpdateTree(id: ModifierId, spendOrReceive: Either[ErgoBox,IndexedErgoBox]): Int = { + private def findAndUpdateTree(id: ModifierId, spendOrReceive: Either[ErgoBox, IndexedErgoBox]): Int = { cfor(trees.length - 1)(_ >= 0, _ - 1) { i => // loop backwards to test latest modifiers first - if(trees(i).id == id) { // address found in last saveLimit modifiers + if (trees(i).id == id) { // address found in last saveLimit modifiers spendOrReceive match { case Left(box) => trees(i).addTx(globalTxIndex).spendBox(box) // spend box case Right(iEb) => trees(i).addTx(globalTxIndex).addBox(iEb) // receive box @@ -142,27 +152,27 @@ class ExtraIndexer(cacheSettings: CacheSettings) // perform segmentation on big modifiers val addressesLen: Int = trees.length cfor(0)(_ < addressesLen, _ + 1) { i => - if(trees(i).txs.length > segmentTreshold || trees(i).boxes.length > segmentTreshold) trees ++= trees(i).splitToSegments() + if (trees(i).txs.length > segmentTreshold || trees(i).boxes.length > segmentTreshold) trees ++= trees(i).splitToSegments() } // merge all modifiers to an Array, avoids reallocations durin concatenation (++) val all: Array[ExtraIndex] = new Array[ExtraIndex](modCount) val offset: Array[Int] = Array(0, general.length, general.length + boxes.length) cfor(0)(_ < general.length, _ + 1) { i => all(i + offset(0)) = general(i) } - cfor(0)(_ < boxes.length , _ + 1) { i => all(i + offset(1)) = boxes(i) } - cfor(0)(_ < trees.length , _ + 1) { i => all(i + offset(2)) = trees(i) } + cfor(0)(_ < boxes.length, _ + 1) { i => all(i + offset(1)) = boxes(i) } + cfor(0)(_ < trees.length, _ + 1) { i => all(i + offset(2)) = trees(i) } // insert modifiers and progress info to db indexedHeightBuffer.clear() globalTxIndexBuffer.clear() globalBoxIndexBuffer.clear() - historyStorage.insertExtra(Array((IndexedHeightKey , indexedHeightBuffer .putInt (indexedHeight ).array), - (GlobalTxIndexKey , globalTxIndexBuffer .putLong(globalTxIndex ).array), - (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), all) + historyStorage.insertExtra(Array((IndexedHeightKey, indexedHeightBuffer.putInt(indexedHeight).array), + (GlobalTxIndexKey, globalTxIndexBuffer.putLong(globalTxIndex).array), + (GlobalBoxIndexKey, globalBoxIndexBuffer.putLong(globalBoxIndex).array)), all) val end: Long = System.nanoTime() - if(writeLog) + if (writeLog) log.info(s"Processed ${trees.length} ErgoTrees with ${boxes.length} boxes and inserted them to database in ${(end - start) / 1000000D}ms") // clear buffers for next batch @@ -175,12 +185,13 @@ class ExtraIndexer(cacheSettings: CacheSettings) /** * Process a batch of BlockTransactions into memory and occasionally write them to database. + * * @param bt - BlockTransaction to process * @param height - height of the block containing the transactions */ - private def index(bt: BlockTransactions, height: Int): Unit = { + protected def index(bt: BlockTransactions, height: Int): Unit = { - if(caughtUp && height <= indexedHeight) return // do not process older blocks again after caught up (due to actor message queue) + if (caughtUp && height <= indexedHeight) return // do not process older blocks again after caught up (due to actor message queue) var boxCount: Int = 0 @@ -196,10 +207,10 @@ class ExtraIndexer(cacheSettings: CacheSettings) tokens.clear() //process transaction inputs - if(height != 1) { //only after 1st block (skip genesis box) + if (height != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => val boxIndex: Int = findAndSpendBox(tx.inputs(i).boxId, tx.id, height) - if(boxIndex >= 0) findAndUpdateTree(bytesToId(hashErgoTree(boxes(boxIndex).box.ergoTree)), Left(boxes(boxIndex).box)) // spend box and add tx + if (boxIndex >= 0) findAndUpdateTree(bytesToId(hashErgoTree(boxes(boxIndex).box.ergoTree)), Left(boxes(boxIndex).box)) // spend box and add tx } } @@ -213,9 +224,9 @@ class ExtraIndexer(cacheSettings: CacheSettings) findAndUpdateTree(bytesToId(hashErgoTree(box.ergoTree)), Right(boxes(findBox(box.id).get))) // check if box is creating a new token, if yes record it - if(box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) + if (box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => - if(!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) { + if (!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) { general += IndexedTokenSerializer.fromBox(box) } } @@ -231,46 +242,46 @@ class ExtraIndexer(cacheSettings: CacheSettings) log.info(s"Buffered block #$height / $chainHeight [txs: ${bt.txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") - if(caughtUp) { + if (caughtUp) { indexedHeight = height // update height here after caught up with chain - if(modCount >= saveLimit || // modifier limit reached to write to db - history.fullBlockHeight == history.headersHeight) // write to db every block after chain synced + if (modCount >= saveLimit || // modifier limit reached to write to db + history.fullBlockHeight == history.headersHeight) // write to db every block after chain synced saveProgress() - }else - if(modCount >= saveLimit) - saveProgress() // active syncing, write to db after modifier limit + } else if (modCount >= saveLimit) + saveProgress() // active syncing, write to db after modifier limit } /** * Main indexer loop that tries to catch up with the already present blocks in database. */ - private def run(): Unit = { + protected def run(): Unit = { - indexedHeight = ByteBuffer.wrap(history.modifierBytesById(bytesToId(IndexedHeightKey)) .getOrElse(Array.fill[Byte](4){0})).getInt - globalTxIndex = ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalTxIndexKey)) .getOrElse(Array.fill[Byte](8){0})).getLong + indexedHeight = ByteBuffer.wrap(history.modifierBytesById(bytesToId(IndexedHeightKey)).getOrElse(Array.fill[Byte](4){0})).getInt + globalTxIndex = ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalTxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong globalBoxIndex = ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalBoxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong log.info(s"Started extra indexer at height $indexedHeight") - while(indexedHeight < chainHeight && !rollback) { + while (indexedHeight < chainHeight && !rollback) { indexedHeight += 1 index(history.bestBlockTransactionsAt(indexedHeight).get, indexedHeight) } - if(rollback) + if (rollback) caughtUp = true - log.info("Indexer caught up with chain") + log.info("Indexer caught up with chain") } /** * Remove all indexes after given height. + * * @param height - starting height */ - def removeAfter(height: Int): Unit = { + protected def removeAfter(height: Int): Unit = { saveProgress(false) @@ -323,6 +334,18 @@ class ExtraIndexer(cacheSettings: CacheSettings) } +} + + +/** + * Actor that constructs an index of database elements. + * @param cacheSettings - cacheSettings to use for saveLimit size + */ +class ExtraIndexer(cacheSettings: CacheSettings) + extends Actor with ExtraIndexerBase { + + override val saveLimit: Int = cacheSettings.history.extraCacheSize * 10 + override def preStart(): Unit = { context.system.eventStream.subscribe(self, classOf[FullBlockApplied]) context.system.eventStream.subscribe(self, classOf[Rollback]) diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala new file mode 100644 index 0000000000..a2c87842d9 --- /dev/null +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala @@ -0,0 +1,28 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.nodeView.state.StateType +import org.ergoplatform.utils.{ErgoPropertyTest, HistoryTestHelpers} + + +class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase with HistoryTestHelpers { + + override protected val saveLimit: Int = 1 // save every block + + property("extra indexer rollback") { + + _history = generateHistory(verifyTransactions = true, StateType.Utxo, PoPoWBootstrap = false, BlocksToKeep) + val chain: Seq[ErgoFullBlock] = genChain(10, _history) // this does not work + _history = applyChain(_history, chain) + + run() + + // TODO: save balances and numeric indexes + + removeAfter(5) + + // TODO: check balances and numeric indexes + + } + +} From 1298308c3c547adca5fea05dfdf62689523199a8 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Wed, 18 Jan 2023 00:21:22 +0100 Subject: [PATCH 47/90] reworked rollback, todo test --- .../http/api/BlockchainApiRoute.scala | 4 +- .../nodeView/history/extra/ExtraIndexer.scala | 98 ++++++++++--------- .../history/extra/IndexedErgoAddress.scala | 58 ++++++++++- .../history/extra/IndexedErgoBox.scala | 10 +- .../extra/IndexedErgoTransaction.scala | 12 ++- .../nodeView/history/extra/IndexedToken.scala | 5 +- .../extra/ExtraIndexerSpecification.scala | 9 +- 7 files changed, 135 insertions(+), 61 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 34708c75a6..99c951be44 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -26,13 +26,13 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting val settings: RESTApiSettings = ergoSettings.scorexSettings.restApi - val paging: Directive[(Int, Int)] = parameters("offset".as[Int] ? 0, "limit".as[Int] ? 5) + private val paging: Directive[(Int, Int)] = parameters("offset".as[Int] ? 0, "limit".as[Int] ? 5) private val MaxItems = 16384 implicit val ae: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder - val ergoAddress: Directive1[ErgoAddress] = entity(as[String]).flatMap(handleErgoAddress) + private val ergoAddress: Directive1[ErgoAddress] = entity(as[String]).flatMap(handleErgoAddress) private def handleErgoAddress(value: String): Directive1[ErgoAddress] = { ae.fromString(value) match { diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 5dd4ee1af4..038e2be026 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -12,6 +12,7 @@ import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.segmentTreshold import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.hashErgoTree +import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.tokenRegistersSet import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.settings.{Algos, CacheSettings, ChainSettings} import scorex.util.{ModifierId, ScorexLogging, bytesToId} @@ -199,10 +200,7 @@ trait ExtraIndexerBase extends ScorexLogging { cfor(0)(_ < bt.txs.length, _ + 1) { n => val tx: ErgoTransaction = bt.txs(n) - - //process transaction - general += IndexedErgoTransaction(tx.id, height, globalTxIndex) - general += NumericTxIndex(globalTxIndex, tx.id) + val inputs: Array[Long] = Array.ofDim[Long](tx.inputs.length) tokens.clear() @@ -210,7 +208,10 @@ trait ExtraIndexerBase extends ScorexLogging { if (height != 1) { //only after 1st block (skip genesis box) cfor(0)(_ < tx.inputs.size, _ + 1) { i => val boxIndex: Int = findAndSpendBox(tx.inputs(i).boxId, tx.id, height) - if (boxIndex >= 0) findAndUpdateTree(bytesToId(hashErgoTree(boxes(boxIndex).box.ergoTree)), Left(boxes(boxIndex).box)) // spend box and add tx + if (boxIndex >= 0) { // spend box and add tx + findAndUpdateTree(bytesToId(hashErgoTree(boxes(boxIndex).box.ergoTree)), Left(boxes(boxIndex).box)) + inputs(i) = boxes(boxIndex).globalIndex + } } } @@ -224,7 +225,7 @@ trait ExtraIndexerBase extends ScorexLogging { findAndUpdateTree(bytesToId(hashErgoTree(box.ergoTree)), Right(boxes(findBox(box.id).get))) // check if box is creating a new token, if yes record it - if (box.additionalTokens.length > 0 && IndexedTokenSerializer.tokenRegistersSet(box)) + if (tokenRegistersSet(box)) cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => if (!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) { general += IndexedTokenSerializer.fromBox(box) @@ -236,6 +237,10 @@ trait ExtraIndexerBase extends ScorexLogging { } + //process transaction + general += IndexedErgoTransaction(tx.id, height, globalTxIndex, inputs) + general += NumericTxIndex(globalTxIndex, tx.id) + globalTxIndex += 1 } @@ -271,9 +276,13 @@ trait ExtraIndexerBase extends ScorexLogging { index(history.bestBlockTransactionsAt(indexedHeight).get, indexedHeight) } - if (rollback) + saveProgress(false) // flush any remaining data + + if (rollback) { caughtUp = true - log.info("Indexer caught up with chain") + log.info("Stopping indexer to perform rollback") + } else + log.info("Indexer caught up with chain") } /** @@ -284,47 +293,48 @@ trait ExtraIndexerBase extends ScorexLogging { protected def removeAfter(height: Int): Unit = { saveProgress(false) - log.info(s"Rolling back indexes from $indexedHeight to $height") - cfor(indexedHeight)(_ > height, _ - 1) { block => // remove all block parts down to "height" - - val bt: Array[ErgoTransaction] = history.bestBlockTransactionsAt(block).get.txs.toArray - cfor(bt.length - 1)(_ >= 0, _ - 1) { tx => - - val outputs: Array[ErgoBox] = bt(tx).outputs.toArray - cfor(outputs.length - 1)(_ >= 0, _ - 1) { n => // revert outputs from addresses - - val i: Int = findAndUpdateTree(bytesToId(hashErgoTree(outputs(n).ergoTree)), Left(outputs(n))) // remove this boxes ergs/tokens - trees(i).txs.remove(trees(i).txs.length - 1) // remove tx number duplicate - trees(i).boxes.remove(trees(i).boxes.length - 1) // remove box number - + val lastTxToKeep: ErgoTransaction = history.bestBlockTransactionsAt(height).get.txs.last + + // remove all tx indexes + val txTarget: Long = history.typedExtraIndexById[IndexedErgoTransaction](lastTxToKeep.id).get.globalIndex + val txs: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId] + while(globalTxIndex > txTarget) { + val tx: IndexedErgoTransaction = NumericTxIndex.getTxByNumber(history, globalTxIndex).get + tx.inputNums.map(NumericBoxIndex.getBoxByNumber(history, _).get).foreach(iEb => { + if(iEb.box.creationHeight <= height) { // if spending box before branchpoint, undo + iEb.spendingHeightOpt = None + iEb.spendingTxIdOpt = None + val address: IndexedErgoAddress = history.typedExtraIndexById[IndexedErgoAddress](bytesToId(hashErgoTree(iEb.box.ergoTree))).get.addBox(iEb, false) + historyStorage.insertExtra(Array.empty, Array(iEb, address)) } - - boxes ++= bt(tx).inputs.map(x => history.typedExtraIndexById[IndexedErgoBox](bytesToId(x.boxId)).get) - cfor(boxes.length - 1)(_ >= 0, _ - 1) { n => // revert inputs from addresses - - boxes(n).spendingTxIdOpt = None - boxes(n).spendingHeightOpt = None - val i: Int = findAndUpdateTree(bytesToId(hashErgoTree(boxes(n).box.ergoTree)), Right(boxes(n))) // add this boxes ergs/tokens - trees(i).txs.remove(trees(i).txs.length - 1) // remove tx number duplicate - trees(i).boxes.remove(trees(i).boxes.length - 1) // remove box number duplicate - + }) + txs += tx.id // tx by id + txs += bytesToId(NumericTxIndex.indexToBytes(globalTxIndex)) // tx id by number + globalTxIndex -= 1 + } + historyStorage.removeExtra(txs.toArray) + + // remove all box indexes, tokens and address balances + val boxTarget: Long = history.typedExtraIndexById[IndexedErgoBox](bytesToId(lastTxToKeep.outputs.last.id)).get.globalIndex + val toRemove: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId] + while(globalBoxIndex > boxTarget) { + val iEb: IndexedErgoBox = NumericBoxIndex.getBoxByNumber(history, globalBoxIndex).get + val address: IndexedErgoAddress = history.typedExtraIndexById[IndexedErgoAddress](bytesToId(hashErgoTree(iEb.box.ergoTree))).get + if(!iEb.trackedBox.isSpent) // remove unspent assests from address + address.spendBox(iEb.box) + if(tokenRegistersSet(iEb.box)) + history.typedExtraIndexById[IndexedToken](IndexedTokenSerializer.fromBox(iEb.box).id) match { + case Some(token) => toRemove += token.id // token created, delete + case None => // no token created } - - historyStorage.removeExtra(outputs.map(x => bytesToId(x.id))) // remove all boxes in tx - - } - - // save updated height and indexes - - indexedHeight = block - globalTxIndex = history.typedExtraIndexById[IndexedErgoTransaction](bt.last.id).get.globalIndex - globalBoxIndex = history.typedExtraIndexById[IndexedErgoBox](bytesToId(bt.last.outputs.last.id)).get.globalIndex - saveProgress(false) - - historyStorage.removeExtra(bt.map(_.id)) // remove all txs in block + address.rollback(txTarget, boxTarget)(_history) + toRemove += iEb.id // box by id + toRemove += bytesToId(NumericBoxIndex.indexToBytes(globalBoxIndex)) // box id by number + globalBoxIndex -= 1 } + historyStorage.removeExtra(toRemove.toArray) log.info(s"Successfully rolled back indexes to $height") diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 165b6b99db..d8e6305bdd 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -1,7 +1,7 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox -import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getBoxes, getSegmentsForRange, getTxs, segmentTreshold, slice} import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, txSegmentId} @@ -12,7 +12,7 @@ import scorex.util.{ModifierId, ScorexLogging, bytesToId} import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.ErgoTree -import scala.collection.mutable.ListBuffer +import scala.collection.mutable.{ArrayBuffer, ListBuffer} import spire.syntax.all.cfor /** @@ -113,10 +113,11 @@ case class IndexedErgoAddress(treeHash: ModifierId, /** * Associate box with this address and update BalanceInfo * @param iEb - box to use + * @param record - whether to add box to boxes list, used in rollbacks (true by default) * @return this address */ - private[extra] def addBox(iEb: IndexedErgoBox): IndexedErgoAddress = { - boxes += iEb.globalIndex + private[extra] def addBox(iEb: IndexedErgoBox, record: Boolean = true): IndexedErgoAddress = { + if(record) boxes += iEb.globalIndex balanceInfo.get.add(iEb.box) this } @@ -131,6 +132,55 @@ case class IndexedErgoAddress(treeHash: ModifierId, this } + /** + * Rollback the state of this address and of boxes associted with it + * @param txTarget - remove transaction numbers above this number + * @param boxTarget - remove box numbers above this number and revert the balance + * @param _history - history handle to update address in database + */ + private[extra] def rollback(txTarget: Long, boxTarget: Long)(_history: ErgoHistory): Unit = { + + def history: ErgoHistoryReader = _history.getReader + + val toSave: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex] + val toRemove: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId] + + // filter tx numbers + do { + val tmp = txs.takeWhile(_ <= txTarget) + txs.clear() + txs ++= tmp + if(txs.isEmpty && txSegmentCount > 0) { // entire current tx set removed, retrieving more from database if possible + val id = txSegmentId(treeHash, txSegmentCount - 1) + txs ++= history.typedExtraIndexById[IndexedErgoAddress](id).get.txs + toRemove += id + txSegmentCount -= 1 + } + }while(txCount() > 0 && txs.last > txTarget) + + // filter box numbers + do { + val tmp = boxes.takeWhile(_ <= boxTarget) + boxes.clear() + boxes ++= tmp + if(boxes.isEmpty && boxSegmentCount > 0) { // entire current box set removed, retrieving more from database if possible + val id = boxSegmentId(treeHash, boxSegmentCount - 1) + boxes ++= history.typedExtraIndexById[IndexedErgoAddress](id).get.boxes + toRemove += id + boxSegmentCount -= 1 + } + }while(boxCount() > 0 && boxes.last > boxTarget) + + if(txCount() == 0 && boxCount() == 0) + toRemove += this.id // address is empty after rollback, delete + else + toSave += this // save the changes made to this address + + _history.historyStorage.insertExtra(Array.empty, toSave.toArray) + _history.historyStorage.removeExtra(toRemove.toArray) + + } + /** * Create an array addresses each containing a "segmentTreshold" number of this address's transaction and box indexes. * These special addresses have their ids calculated by "txSegmentId" and "boxSegmentId" respectively. diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala index f6087d2c31..d1a2d99ee8 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -13,11 +13,11 @@ import sigmastate.Values.ErgoTree /** * Index of a box. - * @param inclusionHeight - height of the block in which the creating transaction was included in - * @param spendingTxIdOpt - optional, id of the spending transaction - * @param spendingHeightOpt - optional, height of the block in which the spending transaction was included in - * @param box - underlying ErgoBox - * @param globalIndex - numeric index of the box + * @param inclusionHeight - height of the block in which the creating transaction was included in + * @param spendingTxIdOpt - optional, id of the spending transaction + * @param spendingHeightOpt - optional, height of the block in which the spending transaction was included in + * @param box - underlying ErgoBox + * @param globalIndex - numeric index of the box */ class IndexedErgoBox(val inclusionHeight: Int, var spendingTxIdOpt: Option[ModifierId], diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index 3e1d489900..61ce46457d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -9,16 +9,19 @@ import scorex.core.serialization.ScorexSerializer import scorex.core.ModifierTypeId import scorex.util.serialization.{Reader, Writer} import scorex.util.{ModifierId, bytesToId} +import spire.implicits.cfor /** * Index of a transaction. * @param txid - id of this transaction * @param height - height of the block which includes this transaction * @param globalIndex - numeric index of this transaction + * @param inputNums - list of transaction inputs (needed for rollback) */ case class IndexedErgoTransaction(txid: ModifierId, height: Int, - globalIndex: Long) extends ExtraIndex { + globalIndex: Long, + inputNums: Array[Long]) extends ExtraIndex { override def id: ModifierId = txid override def serializedId: Array[Byte] = fastIdToBytes(txid) @@ -74,6 +77,8 @@ object IndexedErgoTransactionSerializer extends ScorexSerializer[IndexedErgoTran w.putBytes(iTx.serializedId) w.putInt(iTx.height) w.putLong(iTx.globalIndex) + w.putUShort(iTx.inputNums.length) + cfor(0)(_ < iTx.inputs.length, _ + 1) { i => w.putLong(iTx.inputNums(i)) } } override def parse(r: Reader): IndexedErgoTransaction = { @@ -81,7 +86,10 @@ object IndexedErgoTransactionSerializer extends ScorexSerializer[IndexedErgoTran val id = bytesToId(r.getBytes(idLen)) val height = r.getInt() val globalIndex = r.getLong() - IndexedErgoTransaction(id, height, globalIndex) + val inputCount: Int = r.getUShort() + val inputNums: Array[Long] = Array.ofDim[Long](inputCount) + cfor(0)(_ < inputCount, _ + 1) { i => inputNums(i) = r.getLong() } + IndexedErgoTransaction(id, height, globalIndex, inputNums) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 31582583a1..fc58cf714a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -49,6 +49,9 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { */ def tokenRegistersSet(box: ErgoBox): Boolean = { + // box has tokens + if(box.additionalTokens.length == 0) return false + // registers exist if(!box.additionalRegisters.contains(R4) || !box.additionalRegisters.contains(R5) || @@ -74,7 +77,7 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { * @param reg - register to extract decimals from * @return number of decimals places */ - def getDecimals(reg: EvaluatedValue[_ <: SType]): Int = { + private def getDecimals(reg: EvaluatedValue[_ <: SType]): Int = { try { new String(reg.asInstanceOf[CollectionConstant[SByte.type]].value.toArray, "UTF-8").toInt }catch { diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala index a2c87842d9..1ea47022a2 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala @@ -1,7 +1,7 @@ package org.ergoplatform.nodeView.history.extra -import org.ergoplatform.modifiers.ErgoFullBlock -import org.ergoplatform.nodeView.state.StateType +//import org.ergoplatform.modifiers.ErgoFullBlock +//import org.ergoplatform.nodeView.state.StateType import org.ergoplatform.utils.{ErgoPropertyTest, HistoryTestHelpers} @@ -10,7 +10,7 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w override protected val saveLimit: Int = 1 // save every block property("extra indexer rollback") { - + /** _history = generateHistory(verifyTransactions = true, StateType.Utxo, PoPoWBootstrap = false, BlocksToKeep) val chain: Seq[ErgoFullBlock] = genChain(10, _history) // this does not work _history = applyChain(_history, chain) @@ -22,6 +22,9 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w removeAfter(5) // TODO: check balances and numeric indexes + **/ + + true } From 469e494a758d785c7eda62d37698b16f4ca7ded0 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Wed, 18 Jan 2023 00:56:17 +0100 Subject: [PATCH 48/90] small fix --- .../nodeView/history/extra/ExtraIndexer.scala | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 038e2be026..d3d86ab91a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -148,6 +148,8 @@ trait ExtraIndexerBase extends ScorexLogging { */ private def saveProgress(writeLog: Boolean = true): Unit = { + if(modCount == 0) return + val start: Long = System.nanoTime() // perform segmentation on big modifiers @@ -192,7 +194,9 @@ trait ExtraIndexerBase extends ScorexLogging { */ protected def index(bt: BlockTransactions, height: Int): Unit = { - if (caughtUp && height <= indexedHeight) return // do not process older blocks again after caught up (due to actor message queue) + if (rollback || // rollback in progress + (caughtUp && height <= indexedHeight)) // do not process older blocks again after caught up (due to actor message queue) + return var boxCount: Int = 0 @@ -245,7 +249,7 @@ trait ExtraIndexerBase extends ScorexLogging { } - log.info(s"Buffered block #$height / $chainHeight [txs: ${bt.txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") + log.info(s"Buffered block $height / $chainHeight [txs: ${bt.txs.length}, boxes: $boxCount] (buffer: $modCount / $saveLimit)") if (caughtUp) { @@ -278,15 +282,17 @@ trait ExtraIndexerBase extends ScorexLogging { saveProgress(false) // flush any remaining data - if (rollback) { - caughtUp = true + if (rollback) log.info("Stopping indexer to perform rollback") - } else + else { + caughtUp = true log.info("Indexer caught up with chain") + } + } /** - * Remove all indexes after given height. + * Remove all indexes after a given height and revert address balances. * * @param height - starting height */ From 8ce625fc4381981d38a2f78944cfb094430e6cd8 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sat, 21 Jan 2023 02:49:50 +0100 Subject: [PATCH 49/90] finally done --- .../nodeView/history/extra/ExtraIndexer.scala | 22 +- .../history/extra/IndexedErgoAddress.scala | 2 + .../extra/IndexedErgoTransaction.scala | 2 +- .../extra/ExtraIndexerSpecification.scala | 285 +++++++++++++++++- 4 files changed, 289 insertions(+), 22 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index d3d86ab91a..702269099a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -158,7 +158,7 @@ trait ExtraIndexerBase extends ScorexLogging { if (trees(i).txs.length > segmentTreshold || trees(i).boxes.length > segmentTreshold) trees ++= trees(i).splitToSegments() } - // merge all modifiers to an Array, avoids reallocations durin concatenation (++) + // merge all modifiers to an Array, avoids reallocations during concatenation (++) val all: Array[ExtraIndex] = new Array[ExtraIndex](modCount) val offset: Array[Int] = Array(0, general.length, general.length + boxes.length) cfor(0)(_ < general.length, _ + 1) { i => all(i + offset(0)) = general(i) } @@ -306,30 +306,30 @@ trait ExtraIndexerBase extends ScorexLogging { // remove all tx indexes val txTarget: Long = history.typedExtraIndexById[IndexedErgoTransaction](lastTxToKeep.id).get.globalIndex val txs: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId] + globalTxIndex -= 1 while(globalTxIndex > txTarget) { val tx: IndexedErgoTransaction = NumericTxIndex.getTxByNumber(history, globalTxIndex).get - tx.inputNums.map(NumericBoxIndex.getBoxByNumber(history, _).get).foreach(iEb => { - if(iEb.box.creationHeight <= height) { // if spending box before branchpoint, undo + tx.inputNums.map(NumericBoxIndex.getBoxByNumber(history, _).get).foreach(iEb => { // undo all spendings iEb.spendingHeightOpt = None iEb.spendingTxIdOpt = None val address: IndexedErgoAddress = history.typedExtraIndexById[IndexedErgoAddress](bytesToId(hashErgoTree(iEb.box.ergoTree))).get.addBox(iEb, false) historyStorage.insertExtra(Array.empty, Array(iEb, address)) - } }) txs += tx.id // tx by id txs += bytesToId(NumericTxIndex.indexToBytes(globalTxIndex)) // tx id by number globalTxIndex -= 1 } + globalTxIndex += 1 historyStorage.removeExtra(txs.toArray) // remove all box indexes, tokens and address balances val boxTarget: Long = history.typedExtraIndexById[IndexedErgoBox](bytesToId(lastTxToKeep.outputs.last.id)).get.globalIndex val toRemove: ArrayBuffer[ModifierId] = ArrayBuffer.empty[ModifierId] + globalBoxIndex -= 1 while(globalBoxIndex > boxTarget) { val iEb: IndexedErgoBox = NumericBoxIndex.getBoxByNumber(history, globalBoxIndex).get val address: IndexedErgoAddress = history.typedExtraIndexById[IndexedErgoAddress](bytesToId(hashErgoTree(iEb.box.ergoTree))).get - if(!iEb.trackedBox.isSpent) // remove unspent assests from address - address.spendBox(iEb.box) + address.spendBox(iEb.box) if(tokenRegistersSet(iEb.box)) history.typedExtraIndexById[IndexedToken](IndexedTokenSerializer.fromBox(iEb.box).id) match { case Some(token) => toRemove += token.id // token created, delete @@ -340,13 +340,14 @@ trait ExtraIndexerBase extends ScorexLogging { toRemove += bytesToId(NumericBoxIndex.indexToBytes(globalBoxIndex)) // box id by number globalBoxIndex -= 1 } + globalBoxIndex += 1 historyStorage.removeExtra(toRemove.toArray) + saveProgress(false) + log.info(s"Successfully rolled back indexes to $height") - // restart indexer rollback = false - run() } @@ -378,7 +379,10 @@ class ExtraIndexer(cacheSettings: CacheSettings) case Rollback(branchPoint: ModifierId) => val branchHeight: Int = history.heightOf(branchPoint).get rollback = branchHeight < indexedHeight - if(rollback) removeAfter(branchHeight) + if(rollback) { + removeAfter(branchHeight) + run() // restart indexer + } case Start(history: ErgoHistory) => _history = history diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index d8e6305bdd..c4f421e85d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -140,6 +140,8 @@ case class IndexedErgoAddress(treeHash: ModifierId, */ private[extra] def rollback(txTarget: Long, boxTarget: Long)(_history: ErgoHistory): Unit = { + if(txs.last <= txTarget && boxes.last <= boxTarget) return + def history: ErgoHistoryReader = _history.getReader val toSave: ArrayBuffer[ExtraIndex] = ArrayBuffer.empty[ExtraIndex] diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index 61ce46457d..d1730402a4 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -78,7 +78,7 @@ object IndexedErgoTransactionSerializer extends ScorexSerializer[IndexedErgoTran w.putInt(iTx.height) w.putLong(iTx.globalIndex) w.putUShort(iTx.inputNums.length) - cfor(0)(_ < iTx.inputs.length, _ + 1) { i => w.putLong(iTx.inputNums(i)) } + cfor(0)(_ < iTx.inputNums.length, _ + 1) { i => w.putLong(iTx.inputNums(i)) } } override def parse(r: Reader): IndexedErgoTransaction = { diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala index 1ea47022a2..d717629aa7 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala @@ -1,31 +1,292 @@ package org.ergoplatform.nodeView.history.extra -//import org.ergoplatform.modifiers.ErgoFullBlock -//import org.ergoplatform.nodeView.state.StateType -import org.ergoplatform.utils.{ErgoPropertyTest, HistoryTestHelpers} +import org.ergoplatform.{ErgoAddress, ErgoBox, ErgoBoxCandidate, ErgoScriptPredef, P2PKAddress, UnsignedInput} +import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.mining.difficulty.RequiredDifficulty +import org.ergoplatform.mining.{AutolykosPowScheme, CandidateBlock, CandidateGenerator} +import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionCandidate} +import org.ergoplatform.modifiers.history.header.Header +import org.ergoplatform.modifiers.history.popow.NipopowAlgos +import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction} +import org.ergoplatform.nodeView.history.ErgoHistory +import org.ergoplatform.nodeView.mempool.ErgoMemPool.SortingOption +import org.ergoplatform.nodeView.state.{ErgoState, ErgoStateContext, StateConstants, StateType, UtxoState, UtxoStateReader} +import org.ergoplatform.settings.{ErgoSettings, NetworkType, NodeConfigurationSettings} +import org.ergoplatform.utils.{ErgoPropertyTest, ErgoTestHelpers, HistoryTestHelpers} +import scorex.util.{ModifierId, bytesToId} +import sigmastate.Values +import sigmastate.basics.DLogProtocol.ProveDlog +import spire.implicits.cfor +import java.io.File +import scala.annotation.tailrec +import scala.collection.mutable +import scala.concurrent.duration.{DurationInt, FiniteDuration} +import scala.util.Try class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase with HistoryTestHelpers { override protected val saveLimit: Int = 1 // save every block + val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(StateType.Utxo, verifyTransactions = true, + -1, poPoWBootstrap = false, ChainGenerator.minimalSuffix, mining = false, ChainGenerator.txCostLimit, ChainGenerator.txSizeLimit, useExternalMiner = false, + internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false, + 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 20, + 1000000, 100, adProofsSuffixLength = 112 * 1024, extraIndex = false) + + val HEIGHT: Int = 30 + val BRANCHPOINT: Int = HEIGHT / 2 + property("extra indexer rollback") { - /** - _history = generateHistory(verifyTransactions = true, StateType.Utxo, PoPoWBootstrap = false, BlocksToKeep) - val chain: Seq[ErgoFullBlock] = genChain(10, _history) // this does not work - _history = applyChain(_history, chain) + + val dir: File = createTempDir + dir.mkdirs() + + val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, initSettings.chainSettings, + nodeSettings, settings.scorexSettings, settings.walletSettings, settings.cacheSettings) + + _history = ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + + ChainGenerator.generate(HEIGHT, dir)(_history) + + ExtraIndexerRef.setAddressEncoder(initSettings.chainSettings.addressEncoder) run() - // TODO: save balances and numeric indexes + val txIndexBefore = globalTxIndex + val boxIndexBefore = globalBoxIndex + + var txsIndexed: Int = 0 + var boxesIndexed: Int = 0 + + // manually count balances + val addresses: mutable.HashMap[ErgoAddress,Long] = mutable.HashMap[ErgoAddress,Long]() + cfor(1)(_ <= BRANCHPOINT, _ + 1) { i => + _history.getReader.bestBlockTransactionsAt(i).get.txs.foreach(tx => { txsIndexed += 1 + if(i != 1) { + tx.inputs.foreach(input => { + val iEb: IndexedErgoBox = _history.getReader.typedExtraIndexById[IndexedErgoBox](bytesToId(input.boxId)).get + val address: ErgoAddress = iEb.getAddress + addresses.put(address, addresses(address) - iEb.box.value) + }) + } + tx.outputs.foreach(output => { boxesIndexed += 1 + val address: ErgoAddress = IndexedErgoBoxSerializer.getAddress(output.ergoTree) + addresses.put(address, addresses.getOrElse[Long](address, 0) + output.value) + }) + }) + } + + removeAfter(BRANCHPOINT) + + var mismatches: Int = 0 + + addresses.foreach(e => { + _history.getReader.typedExtraIndexById[IndexedErgoAddress](bytesToId(IndexedErgoAddressSerializer.hashErgoTree(e._1.script))) match { + case Some(iEa) => + if(iEa.balanceInfo.get.nanoErgs != e._2) { + mismatches += 1 + System.err.println(s"Address ${e._1.toString} has ${iEa.balanceInfo.get.nanoErgs / 1000000000}ERG, ${e._2 / 1000000000}ERG expected") + } + case None => + if(e._2 != 0) { + mismatches += 1 + System.err.println(s"Address ${e._1.toString} should exist, but was not found") + } + } + }) + + // indexnumbers + globalTxIndex shouldBe txsIndexed + globalBoxIndex shouldBe boxesIndexed + + // txs + cfor(0)(_ < txIndexBefore, _ + 1) {txNum => + val txOpt = history.typedExtraIndexById[NumericTxIndex](bytesToId(NumericTxIndex.indexToBytes(txNum))) + if(txNum < globalTxIndex) + txOpt shouldNot be(empty) + else + txOpt shouldBe None + } + + // boxes + cfor(0)(_ < boxIndexBefore, _ + 1) { boxNum => + val boxOpt = history.typedExtraIndexById[NumericBoxIndex](bytesToId(NumericBoxIndex.indexToBytes(boxNum))) + if (boxNum < globalBoxIndex) + boxOpt shouldNot be(empty) + else + boxOpt shouldBe None + } + + // balances + mismatches shouldBe 0 + + } + +} + +object ChainGenerator extends ErgoTestHelpers { + + val pow: AutolykosPowScheme = new AutolykosPowScheme(powScheme.k, powScheme.n) + val blockInterval: FiniteDuration = 2.minute + val EmissionTxCost: Long = 20000 + val MinTxAmount: Long = 2000000 + val RewardDelay: Int = initSettings.chainSettings.monetary.minerRewardDelay + val MaxTxsPerBlock: Int = 10 + val minerPk: ProveDlog = defaultProver.hdKeys.head.publicImage + val selfAddressScript: Values.ErgoTree = P2PKAddress(minerPk).script + val minerProp: Values.ErgoTree = ErgoScriptPredef.rewardOutputScript(RewardDelay, minerPk) + val votingEpochLength: Height = votingSettings.votingLength + val protocolVersion: Byte = initSettings.chainSettings.protocolVersion + val minimalSuffix = 2 + val txCostLimit: Height = initSettings.nodeSettings.maxTransactionCost + val txSizeLimit: Height = initSettings.nodeSettings.maxTransactionSize + + var startTime: Long = 0 + + def generate(length: Int, dir: File)(history: ErgoHistory): Unit = { + val stateDir = new File(s"${dir.getAbsolutePath}/state") + stateDir.mkdirs() + val (state, _) = ErgoState.generateGenesisUtxoState(stateDir, StateConstants(initSettings)) + System.out.println(s"Going to generate a chain at ${dir.getAbsolutePath} starting from ${history.bestFullBlockOpt}") + startTime = timeProvider.time - (blockInterval * (length - 1)).toMillis + val chain = loop(state, None, None, Seq())(history) + System.out.println(s"Chain of length ${chain.length} generated") + history.bestHeaderOpt shouldBe history.bestFullBlockOpt.map(_.header) + history.bestFullBlockOpt.get.id shouldBe chain.last + System.out.println("History was generated successfully") + } + + @tailrec + private def loop(state: UtxoState, + initBox: Option[ErgoBox], + last: Option[Header], + acc: Seq[ModifierId])(history: ErgoHistory): Seq[ModifierId] = { + val time: Long = last.map(_.timestamp + blockInterval.toMillis).getOrElse(startTime) + if (time < timeProvider.time) { + val (txs, lastOut) = genTransactions(last.map(_.height).getOrElse(ErgoHistory.GenesisHeight), + initBox, state.stateContext) + + val candidate = genCandidate(defaultProver.hdPubKeys.head.key, last, time, txs, state)(history) + val block = proveCandidate(candidate.get) + + history.append(block.header).get + block.blockSections.foreach(s => if (!history.contains(s)) history.append(s).get) + + val outToPassNext = if (last.isEmpty) { + block.transactions.flatMap(_.outputs).find(_.ergoTree == minerProp) + } else { + lastOut + } + + assert(outToPassNext.isDefined) + + log.info( + s"Block ${block.id} with ${block.transactions.size} transactions at height ${block.header.height} generated") + + loop(state.applyModifier(block, None)(_ => ()).get, outToPassNext, Some(block.header), acc :+ block.id)(history) + } else { + acc + } + } + + private def genTransactions(height: Height, + inOpt: Option[ErgoBox], + ctx: ErgoStateContext): (Seq[ErgoTransaction], Option[ErgoBox]) = { + inOpt + .find { bx => + val canUnlock = (bx.creationHeight + RewardDelay <= height) || (bx.ergoTree != minerProp) + canUnlock && bx.ergoTree != initSettings.chainSettings.monetary.emissionBoxProposition && bx.value >= MinTxAmount + } + .map { input => + val qty = MaxTxsPerBlock + val amount = input.value + val outs = (0 until qty).map(_ => new ErgoBoxCandidate(amount, selfAddressScript, height)) + val x = outs + .foldLeft((Seq.empty[ErgoTransaction], input)) { case ((acc, in), out) => + val inputs = IndexedSeq(in) + val unsignedTx = UnsignedErgoTransaction( + inputs.map(_.id).map(id => new UnsignedInput(id)), + IndexedSeq(out) + ) + + defaultProver.sign(unsignedTx, inputs, emptyDataBoxes, ctx) + .fold(_ => acc -> in, tx => (acc :+ ErgoTransaction(tx)) -> unsignedTx.outputs.head) + } + ._1 + (x, Some(x.last.outputs.head)) + } + .getOrElse(Seq.empty -> inOpt) + } + + private def genCandidate(minerPk: ProveDlog, + lastHeaderOpt: Option[Header], + ts: Long, + txsFromPool: Seq[ErgoTransaction], + state: UtxoStateReader)(history: ErgoHistory): Try[CandidateBlock] = Try { + val stateContext = state.stateContext + val nBits: Long = lastHeaderOpt + .map(parent => history.requiredDifficultyAfter(parent)) + .map(d => RequiredDifficulty.encodeCompactBits(d)) + .getOrElse(settings.chainSettings.initialNBits) + + val interlinks = lastHeaderOpt + .flatMap { h => + history.typedModifierById[Extension](h.extensionId) + .flatMap(ext => NipopowAlgos.unpackInterlinks(ext.fields).toOption) + .map(nipopowAlgos.updateInterlinks(h, _)) + } + .getOrElse(Seq.empty) + val interlinksExtension = nipopowAlgos.interlinksToExtension(interlinks) + + val (extensionCandidate, votes: Array[Byte], version: Byte) = lastHeaderOpt.map { header => + val newHeight = header.height + 1 + val currentParams = stateContext.currentParameters + val betterVersion = protocolVersion > header.version + val votingFinishHeight: Option[Height] = currentParams.softForkStartingHeight + .map(_ + votingSettings.votingLength * votingSettings.softForkEpochs) + val forkVotingAllowed = votingFinishHeight.forall(fh => newHeight < fh) + val forkOrdered = settings.votingTargets.softFork != 0 + val voteForFork = betterVersion && forkOrdered && forkVotingAllowed + + if (newHeight % votingEpochLength == 0 && newHeight > 0) { + val (newParams, _) = currentParams.update(newHeight, voteForFork, stateContext.votingData.epochVotes, emptyVSUpdate, votingSettings) + (newParams.toExtensionCandidate ++ interlinksExtension, + newParams.suggestVotes(settings.votingTargets.targets, voteForFork), + newParams.blockVersion) + } else { + (nipopowAlgos.interlinksToExtension(interlinks), + currentParams.vote(settings.votingTargets.targets, stateContext.votingData.epochVotes, voteForFork), + currentParams.blockVersion) + } + }.getOrElse((interlinksExtension, Array(0: Byte, 0: Byte, 0: Byte), Header.InitialVersion)) - removeAfter(5) + val emissionTxOpt = CandidateGenerator.collectEmission(state, minerPk, emptyStateContext) + val txs = emissionTxOpt.toSeq ++ txsFromPool - // TODO: check balances and numeric indexes - **/ + state.proofsForTransactions(txs).map { case (adProof, adDigest) => + CandidateBlock(lastHeaderOpt, version, nBits, adDigest, adProof, txs, ts, extensionCandidate, votes) + } + }.flatten - true + @tailrec + private def proveCandidate(candidate: CandidateBlock): ErgoFullBlock = { + log.info(s"Trying to prove block with parent ${candidate.parentOpt.map(_.encodedId)} and timestamp ${candidate.timestamp}") + pow.proveCandidate(candidate, defaultProver.hdKeys.head.privateInput.w) match { + case Some(fb) => fb + case _ => + val interlinks = candidate.parentOpt + .map(nipopowAlgos.updateInterlinks(_, NipopowAlgos.unpackInterlinks(candidate.extension.fields).get)) + .getOrElse(Seq.empty) + val minerTag = scorex.utils.Random.randomBytes(Extension.FieldKeySize) + proveCandidate { + candidate.copy( + extension = ExtensionCandidate(Seq(Array(0: Byte, 2: Byte) -> minerTag)) ++ nipopowAlgos.interlinksToExtension(interlinks) + ) + } + } } } From 78b8ebb53dc77840a26343edc5c0bdf1ff1e199f Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sun, 22 Jan 2023 19:26:31 +0100 Subject: [PATCH 50/90] Added indexed height endpoint to monitor progress --- src/main/resources/api/openapi.yaml | 45 ++++++++++++++----- .../http/api/BlockchainApiRoute.scala | 19 +++++++- .../nodeView/history/extra/ExtraIndexer.scala | 11 +++-- 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 8562b8a3f4..c7e863fda2 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -5648,6 +5648,27 @@ paths: schema: $ref: '#/components/schemas/ApiError' + /blockchain/indexedHeight: + get: + summary: Get current block height the indexer is at + operationId: getIndexedHeight + tags: + - blockchain + responses: + '200': + description: height of the indexer and full height + content: + application/json: + schema: + properties: + indexedHeight: + type: integer + default: 0 + description: number of blocks indexed + fullHeight: + type: integer + description: number of all known blocks + /blockchain/transaction/byId/{txId}: get: summary: Retrieve a transaction by its id @@ -5806,12 +5827,12 @@ paths: description: Array of transaction ids items: $ref: '#/components/schemas/ModifierId' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' /blockchain/box/byId/{boxId}: get: @@ -6024,12 +6045,12 @@ paths: description: Array of box ids items: $ref: '#/components/schemas/ModifierId' - default: - description: Error - content: - application/json: - schema: - $ref: '#/components/schemas/ApiError' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' /blockchain/box/byErgoTree: post: diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 99c951be44..8c29cc051f 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -3,6 +3,8 @@ package org.ergoplatform.http.api import akka.actor.{ActorRef, ActorRefFactory} import akka.http.scaladsl.server.{Directive, Directive1, Route, ValidationRejection} import akka.pattern.ask +import io.circe.Json +import io.circe.syntax.EncoderOps import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} import org.ergoplatform.nodeView.ErgoReadersHolder.{GetDataFromHistory, GetReaders, Readers} import org.ergoplatform.nodeView.history.ErgoHistoryReader @@ -42,7 +44,8 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } override val route: Route = pathPrefix("blockchain") { - getTxByIdR ~ + getIndexedHeightR ~ + getTxByIdR ~ getTxByIndexR ~ getTxsByAddressR ~ getTxRangeR ~ @@ -75,11 +78,23 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting case None => None } - private def getTxByIdF(id: ModifierId) : Future[Option[IndexedErgoTransaction]] = + private def getTxByIdF(id: ModifierId): Future[Option[IndexedErgoTransaction]] = getHistory.map { history => getTxById(id)(history) } + private def getIndexedHeightF: Future[Json] = + getHistory.map { history => + Json.obj( + "indexedHeight" -> ExtraIndexerRef.getIndex(ExtraIndexerRef.IndexedHeightKey)(history).getInt().asJson, + "fullHeight" -> history.fullBlockHeight.asJson + ) + } + + private def getIndexedHeightR: Route = (pathPrefix("indexedHeight") & get) { + ApiResponse(getIndexedHeightF) + } + private def getTxByIdR: Route = (get & pathPrefix("transaction" / "byId") & modifierId) { id => ApiResponse(getTxByIdF(id)) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 702269099a..8004b52869 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -7,7 +7,7 @@ import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.{FullBlockApplied, Rollback} -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{GlobalBoxIndexKey, GlobalTxIndexKey, IndexedHeightKey} +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{GlobalBoxIndexKey, GlobalTxIndexKey, IndexedHeightKey, getIndex} import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.segmentTreshold @@ -269,9 +269,9 @@ trait ExtraIndexerBase extends ScorexLogging { */ protected def run(): Unit = { - indexedHeight = ByteBuffer.wrap(history.modifierBytesById(bytesToId(IndexedHeightKey)).getOrElse(Array.fill[Byte](4){0})).getInt - globalTxIndex = ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalTxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong - globalBoxIndex = ByteBuffer.wrap(history.modifierBytesById(bytesToId(GlobalBoxIndexKey)).getOrElse(Array.fill[Byte](8){0})).getLong + indexedHeight = getIndex(IndexedHeightKey)(history).getInt + globalTxIndex = getIndex(GlobalTxIndexKey)(history).getLong + globalBoxIndex = getIndex(GlobalBoxIndexKey)(history).getLong log.info(s"Started extra indexer at height $indexedHeight") @@ -433,6 +433,9 @@ object ExtraIndexerRef { val GlobalTxIndexKey: Array[Byte] = Algos.hash("txns height") val GlobalBoxIndexKey: Array[Byte] = Algos.hash("boxes height") + def getIndex(key: Array[Byte])(history: ErgoHistoryReader): ByteBuffer = + ByteBuffer.wrap(history.modifierBytesById(bytesToId(key)).getOrElse(Array.fill[Byte](8){0})) + def apply(chainSettings: ChainSettings, cacheSettings: CacheSettings)(implicit system: ActorSystem): ActorRef = { val actor = system.actorOf(Props.create(classOf[ExtraIndexer], cacheSettings)) _ae = chainSettings.addressEncoder From c6934a4bb86d2c46395e3b5aabf332e9583de6cf Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Tue, 24 Jan 2023 00:49:27 +0300 Subject: [PATCH 51/90] Application removed --- .../crypto/authds/benchmarks/Helper.scala | 2 +- .../scala/scorex/db/LDBVersionedStore.scala | 7 +- .../batch/benchmark/BatchingBenchmark.scala | 2 +- .../scala/scorex/core/app/Application.scala | 157 ------------------ 4 files changed, 7 insertions(+), 161 deletions(-) delete mode 100644 src/main/scala/scorex/core/app/Application.scala diff --git a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala index a3e1ead7af..2648c36245 100644 --- a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala +++ b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala @@ -12,7 +12,7 @@ object Helper { type HF = Blake2b256.type type Prover = PersistentBatchAVLProver[Digest32, HF] - implicit val hf = Blake2b256 + implicit val hf: HF = Blake2b256 val kl = 32 val vl = 8 diff --git a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala b/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala index 63dba526e2..2791f8f03c 100644 --- a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala +++ b/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala @@ -228,9 +228,12 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e Undo(versionID, key, value) } + /** + * Write versioned batch update to the database, removing keys from the database and adding new key -> value pairs + */ def update(versionID: VersionID, - toRemove: Iterable[Array[Byte]], - toUpdate: Iterable[(Array[Byte], Array[Byte])]): Try[Unit] = Try { + toRemove: TraversableOnce[Array[Byte]], + toUpdate: TraversableOnce[(Array[Byte], Array[Byte])]): Try[Unit] = Try { lock.writeLock().lock() val lastLsn = lsn // remember current LSN value val batch = db.createWriteBatch() diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala index ecadcd1ad5..1067650c2f 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala @@ -15,7 +15,7 @@ object BatchingBenchmark extends App with FileHelper { val NumMods = 200000 - implicit val hf = Blake2b256 + implicit val hf: HF = Blake2b256 type HF = Blake2b256.type val store = new LDBVersionedStore(getRandomTempDir, initialKeepVersions = 10) diff --git a/src/main/scala/scorex/core/app/Application.scala b/src/main/scala/scorex/core/app/Application.scala deleted file mode 100644 index a7224b5791..0000000000 --- a/src/main/scala/scorex/core/app/Application.scala +++ /dev/null @@ -1,157 +0,0 @@ -package scorex.core.app - -import akka.actor.{ActorRef, ActorSystem} -import akka.http.scaladsl.Http -import akka.http.scaladsl.server.{ExceptionHandler, RejectionHandler, Route} -import org.ergoplatform.ErgoApp -import org.ergoplatform.nodeView.history.ErgoSyncInfoMessageSpec -import org.ergoplatform.settings.ErgoSettings -import scorex.core.api.http.{ - ApiErrorHandler, - ApiRejectionHandler, - ApiRoute, - CompositeHttpService -} -import scorex.core.network._ -import scorex.core.network.message.Message.MessageCode -import scorex.core.network.message._ -import scorex.core.network.peer.PeerManagerRef -import scorex.core.settings.ScorexSettings -import scorex.core.utils.NetworkTimeProvider -import scorex.util.ScorexLogging - -import java.net.InetSocketAddress -import scala.concurrent.ExecutionContext - -trait Application extends ScorexLogging { - - //settings - val ergoSettings: ErgoSettings - implicit val scorexSettings: ScorexSettings - - //api - val apiRoutes: Seq[ApiRoute] - - implicit def exceptionHandler: ExceptionHandler = ApiErrorHandler.exceptionHandler - implicit def rejectionHandler: RejectionHandler = ApiRejectionHandler.rejectionHandler - - implicit protected lazy val actorSystem: ActorSystem = ActorSystem( - scorexSettings.network.agentName - ) - - implicit val executionContext: ExecutionContext = - actorSystem.dispatchers.lookup("scorex.executionContext") - - protected val features: Seq[PeerFeature] - protected val additionalMessageSpecs: Seq[MessageSpec[_]] - - //p2p - private val upnpGateway: Option[UPnPGateway] = - if (scorexSettings.network.upnpEnabled) UPnP.getValidGateway(scorexSettings.network) - else None - // TODO use available port on gateway instead settings.network.bindAddress.getPort - upnpGateway.foreach(_.addPort(scorexSettings.network.bindAddress.getPort)) - - private lazy val basicSpecs = { - Seq( - GetPeersSpec, - new PeersSpec(scorexSettings.network.maxPeerSpecObjects), - InvSpec, - RequestModifierSpec, - ModifiersSpec - ) - } - - val nodeViewHolderRef: ActorRef - val nodeViewSynchronizer: ActorRef - - /** API description in openapi format in YAML or JSON */ - val swaggerConfig: String - - val timeProvider = new NetworkTimeProvider(scorexSettings.ntp) - - //an address to send to peers - lazy val externalSocketAddress: Option[InetSocketAddress] = { - scorexSettings.network.declaredAddress orElse { - // TODO use available port on gateway instead settings.bindAddress.getPort - upnpGateway.map(u => - new InetSocketAddress( - u.externalAddress, - scorexSettings.network.bindAddress.getPort - ) - ) - } - } - - val scorexContext = ScorexContext( - messageSpecs = basicSpecs ++ additionalMessageSpecs, - upnpGateway = upnpGateway, - timeProvider = timeProvider, - externalNodeAddress = externalSocketAddress - ) - - val peerManagerRef = PeerManagerRef(ergoSettings, scorexContext) - - private val messageHandlers: ActorRef => Map[MessageCode, ActorRef] = - networkControllerRef => { - Map( - InvSpec.messageCode -> nodeViewSynchronizer, - RequestModifierSpec.messageCode -> nodeViewSynchronizer, - ModifiersSpec.messageCode -> nodeViewSynchronizer, - ErgoSyncInfoMessageSpec.messageCode -> nodeViewSynchronizer, - PeersSpec.messageCode -> PeerSynchronizerRef( - "PeerSynchronizer", - networkControllerRef, - peerManagerRef, - scorexSettings.network - ) - ) - } - - val networkControllerRef: ActorRef = - NetworkControllerRef( - "networkController", - ergoSettings, - peerManagerRef, - scorexContext, - messageHandlers - ) - - val peerSynchronizer: ActorRef = - PeerSynchronizerRef( - "PeerSynchronizer", - networkControllerRef, - peerManagerRef, - scorexSettings.network - ) - - lazy val combinedRoute: Route = CompositeHttpService( - actorSystem, - apiRoutes, - scorexSettings.restApi, - swaggerConfig - ).compositeRoute - - def run(): Unit = { - val applicationNameLimit: Int = 50 - require(scorexSettings.network.agentName.length <= applicationNameLimit) - - log.debug(s"Available processors: ${Runtime.getRuntime.availableProcessors}") - log.debug(s"Max memory available: ${Runtime.getRuntime.maxMemory}") - log.debug(s"RPC is allowed at ${scorexSettings.restApi.bindAddress.toString}") - - val bindAddress = scorexSettings.restApi.bindAddress - - Http() - .newServerAt(bindAddress.getAddress.getHostAddress, bindAddress.getPort) - .bindFlow(combinedRoute) - - //on unexpected shutdown - Runtime.getRuntime.addShutdownHook(new Thread() { - override def run() { - log.error("Unexpected shutdown") - ErgoApp.shutdownSystem() - } - }) - } -} From 45187041feea8583f8a4547ed6abe9ea7b98eda6 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Tue, 24 Jan 2023 20:48:26 +0300 Subject: [PATCH 52/90] NetworkObjectTypeId --- .../bench/misc/ModifierWriter.scala | 11 ++- .../ergoplatform/local/MempoolAuditor.scala | 5 +- .../modifiers/ErgoFullBlock.scala | 6 +- .../modifiers/NetworkObjectTypeId.scala | 70 +++++++++++++++++++ .../modifiers/NonHeaderBlockSection.scala | 8 +-- .../modifiers/history/ADProofs.scala | 8 +-- .../modifiers/history/BlockTransactions.scala | 6 +- .../history/extension/Extension.scala | 7 +- .../modifiers/history/header/Header.scala | 11 ++- .../modifiers/mempool/ErgoTransaction.scala | 4 +- .../modifiers/state/UTXOSnapshotChunk.scala | 39 ----------- .../state/UTXOSnapshotManifest.scala | 42 ----------- .../network/ErgoNodeViewSynchronizer.scala | 55 +++++++-------- .../nodeView/ErgoNodeViewHolder.scala | 26 ++++--- .../nodeView/history/ErgoHistoryReader.scala | 8 +-- .../history/storage/HistoryStorage.scala | 9 +-- .../ToDownloadProcessor.scala | 17 +++-- .../nodeView/state/UtxoState.scala | 3 +- .../org/ergoplatform/settings/Constants.scala | 10 +-- .../settings/ValidationRules.scala | 9 ++- .../tools/ValidationRulesPrinter.scala | 6 +- .../scala/scorex/core/NodeViewModifier.scala | 8 +-- .../scorex/core/consensus/ProgressInfo.scala | 5 +- src/main/scala/scorex/core/core.scala | 10 +-- .../scorex/core/network/DeliveryTracker.scala | 34 ++++----- .../network/message/BasicMessagesRepo.scala | 11 +-- .../scorex/core/transaction/Transaction.scala | 10 +-- .../core/validation/ModifierError.scala | 10 +-- .../core/validation/ModifierValidator.scala | 35 +++++----- .../core/validation/ValidationSettings.scala | 4 +- .../mempool/ErgoTransactionSpec.scala | 9 ++- .../NonVerifyADHistorySpecification.scala | 10 --- .../VerifyNonADHistorySpecification.scala | 9 ++- .../ErgoTransactionGenerators.scala | 8 +-- .../core/network/DeliveryTrackerSpec.scala | 4 +- .../testkit/generators/ObjectGenerators.scala | 9 +-- 36 files changed, 249 insertions(+), 287 deletions(-) create mode 100644 src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala delete mode 100644 src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotChunk.scala delete mode 100644 src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotManifest.scala diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/misc/ModifierWriter.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/misc/ModifierWriter.scala index 3f448d9125..1372cab088 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/bench/misc/ModifierWriter.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/bench/misc/ModifierWriter.scala @@ -1,18 +1,17 @@ package org.ergoplatform.bench.misc import java.io.{InputStream, OutputStream} - import com.google.common.primitives.Ints import org.ergoplatform.Utils._ -import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.modifiers.{BlockSection, NetworkObjectTypeId} import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} import scorex.core.serialization.ScorexSerializer -import scorex.core.{ModifierTypeId, NodeViewModifier} +import scorex.core.NodeViewModifier object ModifierWriter { - val modifierSerializers: Map[ModifierTypeId, ScorexSerializer[_ <: BlockSection]] = + val modifierSerializers: Map[NetworkObjectTypeId.Value, ScorexSerializer[_ <: BlockSection]] = Map(Header.modifierTypeId -> HeaderSerializer, BlockTransactions.modifierTypeId -> BlockTransactionsSerializer, ADProofs.modifierTypeId -> ADProofsSerializer) @@ -34,9 +33,9 @@ object ModifierWriter { mod <- modifierSerializers(typeId).parseBytesTry(bytes).toOption } yield mod - private def readModId(implicit fis: InputStream): Option[ModifierTypeId] = { + private def readModId(implicit fis: InputStream): Option[NetworkObjectTypeId.Value] = { val int = fis.read() - if (int == -1) { None } else { Some(ModifierTypeId @@ int.toByte) } + if (int == -1) { None } else { Some(NetworkObjectTypeId.fromByte(int.toByte)) } } } diff --git a/src/main/scala/org/ergoplatform/local/MempoolAuditor.scala b/src/main/scala/org/ergoplatform/local/MempoolAuditor.scala index c2239037c2..6578613c5e 100644 --- a/src/main/scala/org/ergoplatform/local/MempoolAuditor.scala +++ b/src/main/scala/org/ergoplatform/local/MempoolAuditor.scala @@ -4,7 +4,7 @@ import akka.actor.SupervisorStrategy.{Restart, Stop} import akka.actor.{Actor, ActorInitializationException, ActorKilledException, ActorRef, ActorRefFactory, DeathPactException, OneForOneStrategy, Props} import org.ergoplatform.local.CleanupWorker.RunCleanup import org.ergoplatform.local.MempoolAuditor.CleanupDone -import org.ergoplatform.modifiers.mempool.UnconfirmedTransaction +import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader import org.ergoplatform.settings.ErgoSettings import scorex.core.network.Broadcast @@ -12,7 +12,6 @@ import scorex.core.network.NetworkController.ReceivableMessages.SendToNetwork import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.RecheckMempool import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoStateReader} import scorex.core.network.message.{InvData, InvSpec, Message} -import scorex.core.transaction.Transaction import scorex.util.ScorexLogging import scala.concurrent.duration._ @@ -87,7 +86,7 @@ class MempoolAuditor(nodeViewHolderRef: ActorRef, private def broadcastTx(unconfirmedTx: UnconfirmedTransaction): Unit = { val msg = Message( InvSpec, - Right(InvData(Transaction.ModifierTypeId, Seq(unconfirmedTx.id))), + Right(InvData(ErgoTransaction.modifierTypeId, Seq(unconfirmedTx.id))), None ) networkControllerRef ! SendToNetwork(msg, Broadcast) diff --git a/src/main/scala/org/ergoplatform/modifiers/ErgoFullBlock.scala b/src/main/scala/org/ergoplatform/modifiers/ErgoFullBlock.scala index 80240799d0..6fb1a0ff43 100644 --- a/src/main/scala/org/ergoplatform/modifiers/ErgoFullBlock.scala +++ b/src/main/scala/org/ergoplatform/modifiers/ErgoFullBlock.scala @@ -8,7 +8,7 @@ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.history.{ADProofs, BlockTransactions} import org.ergoplatform.modifiers.mempool.ErgoTransaction import scorex.core.serialization.ScorexSerializer -import scorex.core.{ModifierTypeId, TransactionsCarryingPersistentNodeViewModifier} +import scorex.core.TransactionsCarryingPersistentNodeViewModifier import scorex.util.ModifierId case class ErgoFullBlock(header: Header, @@ -20,7 +20,7 @@ case class ErgoFullBlock(header: Header, override type M = ErgoFullBlock - override val modifierTypeId: ModifierTypeId = ErgoFullBlock.modifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = ErgoFullBlock.modifierTypeId override def serializedId: Array[Byte] = header.serializedId @@ -49,7 +49,7 @@ case class ErgoFullBlock(header: Header, object ErgoFullBlock extends ApiCodecs { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (-127: Byte) + val modifierTypeId: NetworkObjectTypeId.Value = FullBlockTypeId.value implicit val jsonEncoder: Encoder[ErgoFullBlock] = { b: ErgoFullBlock => Json.obj( diff --git a/src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala b/src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala new file mode 100644 index 0000000000..8e258d96d6 --- /dev/null +++ b/src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala @@ -0,0 +1,70 @@ +package org.ergoplatform.modifiers + +import supertagged.TaggedType +import NetworkObjectTypeId._ + +/** + * Hierarchy encoding blockchain objects being sent over wire: block sections, chunks of UTXO set snapshot etc + */ +sealed trait NetworkObjectTypeId { + /** + * 1-byte ID of network object type + */ + val value: NetworkObjectTypeId.Value +} + +object NetworkObjectTypeId { + object Value extends TaggedType[Byte] + type Value = Value.Type + + @inline + def fromByte(value: Byte): Value = Value @@ value +} + +/** + * Unconfirmed transactions sent outside blocks + */ +object TransactionTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(2) +} + +/** + * Block header, section of a block PoW is done on top of. This section is committing to other sections + */ +object HeaderTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(101) +} + +/** + * Block transactions sections. Contains all the transactions for a block. + */ +object BlockTransactionsTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(102) +} + +/** + * Block section which contains proofs of correctness for UTXO set transformations. + * The section contains proofs for all the transformations (i.e. for all the block transactions) + */ +object ProofsTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(104) +} + + +/** + * Block section which contains key-value pairs with different additional data. + * Interlinks vector (for nipopow proofs) written there, as well as current network parameters + * (at the beginning of voting epoch), but miners can also put arbitrary data there. + */ +object ExtensionTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(108) +} + +/** + * Virtual object which is not being sent over the wire rather, constructed locally from different sections + * got over the wire (header, transactions, extension in the "utxo" mode, those three sections plus proofs in + * the "digest" mode). + */ +object FullBlockTypeId extends NetworkObjectTypeId { + override val value: Value = fromByte(-127) +} diff --git a/src/main/scala/org/ergoplatform/modifiers/NonHeaderBlockSection.scala b/src/main/scala/org/ergoplatform/modifiers/NonHeaderBlockSection.scala index bbed96e1ff..5a18374056 100644 --- a/src/main/scala/org/ergoplatform/modifiers/NonHeaderBlockSection.scala +++ b/src/main/scala/org/ergoplatform/modifiers/NonHeaderBlockSection.scala @@ -1,7 +1,6 @@ package org.ergoplatform.modifiers import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId import scorex.crypto.hash.Digest32 import scorex.util.{ModifierId, bytesToId, idToBytes} @@ -10,7 +9,8 @@ import scorex.util.{ModifierId, bytesToId, idToBytes} */ trait NonHeaderBlockSection extends BlockSection { - override lazy val serializedId: Array[Byte] = NonHeaderBlockSection.computeIdBytes(modifierTypeId, headerId, digest) + override lazy val serializedId: Array[Byte] = + NonHeaderBlockSection.computeIdBytes(modifierTypeId, headerId, digest) override lazy val id: ModifierId = bytesToId(serializedId) @@ -22,9 +22,9 @@ trait NonHeaderBlockSection extends BlockSection { } object NonHeaderBlockSection { - def computeId(modifierType: ModifierTypeId, headerId: ModifierId, digest: Array[Byte]): ModifierId = + def computeId(modifierType: NetworkObjectTypeId.Value, headerId: ModifierId, digest: Array[Byte]): ModifierId = bytesToId(computeIdBytes(modifierType, headerId, digest)) - def computeIdBytes(modifierType: ModifierTypeId, headerId: ModifierId, digest: Array[Byte]): Array[Byte] = + def computeIdBytes(modifierType: NetworkObjectTypeId.Value, headerId: ModifierId, digest: Array[Byte]): Array[Byte] = Algos.hash.prefixedHash(modifierType, idToBytes(headerId), digest) } diff --git a/src/main/scala/org/ergoplatform/modifiers/history/ADProofs.scala b/src/main/scala/org/ergoplatform/modifiers/history/ADProofs.scala index 4471c7da30..dd3dc69966 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/ADProofs.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/ADProofs.scala @@ -3,11 +3,10 @@ package org.ergoplatform.modifiers.history import io.circe.syntax._ import io.circe.{Decoder, Encoder, HCursor} import org.ergoplatform.http.api.ApiCodecs -import org.ergoplatform.modifiers.NonHeaderBlockSection +import org.ergoplatform.modifiers.{NetworkObjectTypeId, NonHeaderBlockSection, ProofsTypeId} import org.ergoplatform.modifiers.state._ import org.ergoplatform.settings.Algos.HF import org.ergoplatform.settings.{Algos, Constants} -import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.crypto.authds.avltree.batch.{Lookup => _, _} import scorex.crypto.authds.{ADDigest, ADValue, SerializedAdProof} @@ -24,7 +23,7 @@ case class ADProofs(headerId: ModifierId, override def digest: Digest32 = ADProofs.proofDigest(proofBytes) - override val modifierTypeId: ModifierTypeId = ADProofs.modifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = ADProofs.modifierTypeId override type M = ADProofs @@ -69,7 +68,8 @@ case class ADProofs(headerId: ModifierId, } object ADProofs extends ApiCodecs { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (104: Byte) + + val modifierTypeId: NetworkObjectTypeId.Value = ProofsTypeId.value val KL = 32 diff --git a/src/main/scala/org/ergoplatform/modifiers/history/BlockTransactions.scala b/src/main/scala/org/ergoplatform/modifiers/history/BlockTransactions.scala index cb926f3894..12ce33b24d 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/BlockTransactions.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/BlockTransactions.scala @@ -3,7 +3,7 @@ package org.ergoplatform.modifiers.history import io.circe.syntax._ import io.circe.{Decoder, Encoder, HCursor} import org.ergoplatform.http.api.ApiCodecs -import org.ergoplatform.modifiers.NonHeaderBlockSection +import org.ergoplatform.modifiers.{BlockTransactionsTypeId, NetworkObjectTypeId, NonHeaderBlockSection} import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.{ErgoTransaction, ErgoTransactionSerializer} import org.ergoplatform.nodeView.mempool.TransactionMembershipProof @@ -37,7 +37,7 @@ case class BlockTransactions(headerId: ModifierId, assert(txs.nonEmpty, "Block should always contain at least 1 coinbase-like transaction") - override val modifierTypeId: ModifierTypeId = BlockTransactions.modifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = BlockTransactions.modifierTypeId /** * Ids of block transactions @@ -94,7 +94,7 @@ case class BlockTransactions(headerId: ModifierId, object BlockTransactions extends ApiCodecs { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (102: Byte) + val modifierTypeId: NetworkObjectTypeId.Value = BlockTransactionsTypeId.value // Used in the miner when a BlockTransaction instance is not generated yet (because a header is not known) def transactionsRoot(txs: Seq[ErgoTransaction], blockVersion: Version): Digest32 = { diff --git a/src/main/scala/org/ergoplatform/modifiers/history/extension/Extension.scala b/src/main/scala/org/ergoplatform/modifiers/history/extension/Extension.scala index 38667c2ebd..2b5e03b2f8 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/extension/Extension.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/extension/Extension.scala @@ -4,9 +4,8 @@ import com.google.common.primitives.Bytes import io.circe.syntax._ import io.circe.{Decoder, Encoder, HCursor} import org.ergoplatform.http.api.ApiCodecs -import org.ergoplatform.modifiers.NonHeaderBlockSection +import org.ergoplatform.modifiers.{ExtensionTypeId, NetworkObjectTypeId, NonHeaderBlockSection} import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.crypto.authds.LeafData import scorex.crypto.authds.merkle.MerkleTree @@ -25,7 +24,7 @@ case class Extension(headerId: ModifierId, override val sizeOpt: Option[Int] = None) extends ExtensionCandidate(fields) with NonHeaderBlockSection { - override val modifierTypeId: ModifierTypeId = Extension.modifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = Extension.modifierTypeId override type M = Extension @@ -55,7 +54,7 @@ object Extension extends ApiCodecs { Algos.merkleTree(LeafData @@ fields.map(kvToLeaf)) } - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (108: Byte) + val modifierTypeId: NetworkObjectTypeId.Value = ExtensionTypeId.value implicit val jsonEncoder: Encoder[Extension] = { e: Extension => Map( diff --git a/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala b/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala index 58919ee96d..25a39f2f9b 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala @@ -7,14 +7,13 @@ import org.ergoplatform.mining.difficulty.RequiredDifficulty import org.ergoplatform.mining.AutolykosSolution import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.{ADProofs, BlockTransactions, PreHeader} -import org.ergoplatform.modifiers.{NonHeaderBlockSection, BlockSection} +import org.ergoplatform.modifiers.{BlockSection, HeaderTypeId, NetworkObjectTypeId, NonHeaderBlockSection} import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.history.ErgoHistory.Difficulty import org.ergoplatform.settings.{Algos, Constants} import org.ergoplatform.wallet.interpreter.ErgoInterpreter import scorex.core.serialization.ScorexSerializer import scorex.core.utils.NetworkTimeProvider -import scorex.core.ModifierTypeId import scorex.crypto.authds.ADDigest import scorex.crypto.hash.Digest32 import scorex.util._ @@ -60,7 +59,7 @@ case class Header(override val version: Header.Version, override type M = Header - override val modifierTypeId: ModifierTypeId = Header.modifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = Header.modifierTypeId lazy val requiredDifficulty: Difficulty = RequiredDifficulty.decodeCompactBits(nBits) @@ -75,7 +74,7 @@ case class Header(override val version: Header.Version, /** * Expected identifiers of the block sections corresponding to this header */ - lazy val sectionIds: Seq[(ModifierTypeId, ModifierId)] = + lazy val sectionIds: Seq[(NetworkObjectTypeId.Value, ModifierId)] = Array( (ADProofs.modifierTypeId, ADProofsId), (BlockTransactions.modifierTypeId, transactionsId), @@ -86,7 +85,7 @@ case class Header(override val version: Header.Version, * Expected identifiers of the block sections corresponding to this header, * except of state transformations proof section id */ - lazy val sectionIdsWithNoProof: Seq[(ModifierTypeId, ModifierId)] = sectionIds.tail + lazy val sectionIdsWithNoProof: Seq[(NetworkObjectTypeId.Value, ModifierId)] = sectionIds.tail override lazy val toString: String = s"Header(${this.asJson.noSpaces})" @@ -157,7 +156,7 @@ object Header extends ApiCodecs { votes = header.votes.toColl ) - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (101: Byte) + val modifierTypeId: NetworkObjectTypeId.Value = HeaderTypeId.value lazy val GenesisParentId: ModifierId = bytesToId(Array.fill(Constants.HashLength)(0: Byte)) diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala index bbf834acf1..983205c8ed 100644 --- a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala @@ -6,7 +6,7 @@ import org.ergoplatform.SigmaConstants.{MaxBoxSize, MaxPropositionBytes} import org.ergoplatform._ import org.ergoplatform.http.api.ApiCodecs import org.ergoplatform.mining.emission.EmissionRules -import org.ergoplatform.modifiers.ErgoNodeViewModifier +import org.ergoplatform.modifiers.{ErgoNodeViewModifier, NetworkObjectTypeId, TransactionTypeId} import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.ErgoTransaction.unresolvedIndices import org.ergoplatform.nodeView.ErgoContext @@ -474,6 +474,8 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input], object ErgoTransaction extends ApiCodecs with ScorexLogging with ScorexEncoding { + val modifierTypeId: NetworkObjectTypeId.Value = TransactionTypeId.value + def apply(inputs: IndexedSeq[Input], outputCandidates: IndexedSeq[ErgoBoxCandidate]): ErgoTransaction = ErgoTransaction(inputs, IndexedSeq.empty, outputCandidates, None) diff --git a/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotChunk.scala b/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotChunk.scala deleted file mode 100644 index caf93c3afd..0000000000 --- a/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotChunk.scala +++ /dev/null @@ -1,39 +0,0 @@ -package org.ergoplatform.modifiers.state - -import org.ergoplatform.ErgoBox -import org.ergoplatform.modifiers.BlockSection -import org.ergoplatform.modifiers.state.UTXOSnapshotChunk.StateElement -import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId -import scorex.core.serialization.ScorexSerializer -import scorex.crypto.authds.LeafData -import scorex.crypto.hash.Digest32 -import scorex.util.{ModifierId, bytesToId} -import scorex.utils.Random - -case class UTXOSnapshotChunk(stateElements: Seq[StateElement], - index: Short) extends BlockSection { - override val modifierTypeId: ModifierTypeId = UTXOSnapshotChunk.modifierTypeId - - //TODO implement correctly - override lazy val id: ModifierId = bytesToId(Random.randomBytes(32)) - - override def parentId: ModifierId = ??? - - override def serializedId: Array[Byte] = ??? - - override type M = UTXOSnapshotChunk - - override def serializer: ScorexSerializer[UTXOSnapshotChunk] = ??? - - lazy val rootHash: Digest32 = Algos.merkleTreeRoot(stateElements.map(LeafData @@ _.bytes)) - - override val sizeOpt: Option[Int] = None - -} - -object UTXOSnapshotChunk { - type StateElement = ErgoBox - - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (107: Byte) -} diff --git a/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotManifest.scala b/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotManifest.scala deleted file mode 100644 index 2730587874..0000000000 --- a/src/main/scala/org/ergoplatform/modifiers/state/UTXOSnapshotManifest.scala +++ /dev/null @@ -1,42 +0,0 @@ -package org.ergoplatform.modifiers.state - -import org.ergoplatform.modifiers.BlockSection -import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId -import scorex.core.serialization.ScorexSerializer -import scorex.core.utils.concatBytes -import scorex.crypto.authds.LeafData -import scorex.crypto.hash.Digest32 -import scorex.util.{ModifierId, bytesToId, idToBytes} - -import scala.util.Try - -case class UTXOSnapshotManifest(chunkRootHashes: Seq[Array[Byte]], blockId: ModifierId) extends BlockSection { - override val modifierTypeId: ModifierTypeId = UTXOSnapshotManifest.modifierTypeId - - override def serializedId: Array[Byte] = Algos.hash(concatBytes(chunkRootHashes :+ idToBytes(blockId))) - - override lazy val id: ModifierId = bytesToId(serializedId) - - override type M = UTXOSnapshotManifest - - override def serializer: ScorexSerializer[UTXOSnapshotManifest] = ??? - - override def parentId: ModifierId = ??? - - lazy val rootHash: Digest32 = Algos.merkleTreeRoot(LeafData @@ chunkRootHashes) - - override val sizeOpt: Option[Int] = None -} - -object UTXOSnapshotManifest { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ (106: Byte) - - def validate(manifest: UTXOSnapshotManifest, header: Header): Try[Unit] = Try { - require(manifest.blockId == header.id) - require(java.util.Arrays.equals(manifest.rootHash, header.stateRoot)) - ??? - } -} - diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 7a94a60ec5..3e7fee5272 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -3,8 +3,8 @@ package org.ergoplatform.network import akka.actor.SupervisorStrategy.{Restart, Stop} import akka.actor.{Actor, ActorInitializationException, ActorKilledException, ActorRef, ActorRefFactory, DeathPactException, OneForOneStrategy, Props} import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.modifiers.mempool.{ErgoTransactionSerializer, UnconfirmedTransaction} -import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.modifiers.mempool.{ErgoTransaction, ErgoTransactionSerializer, UnconfirmedTransaction} +import org.ergoplatform.modifiers.{BlockSection, NetworkObjectTypeId} import org.ergoplatform.nodeView.history.{ErgoSyncInfoV1, ErgoSyncInfoV2} import org.ergoplatform.nodeView.history._ import ErgoNodeViewSynchronizer.{CheckModifiersToDownload, IncomingTxInfo, TransactionProcessingCacheRecord} @@ -16,7 +16,7 @@ import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages._ import org.ergoplatform.nodeView.ErgoNodeViewHolder._ import scorex.core.consensus.{Equal, Fork, Nonsense, Older, Unknown, Younger} import scorex.core.network.ModifiersStatus.Requested -import scorex.core.{ModifierTypeId, NodeViewModifier, idsToString} +import scorex.core.{NodeViewModifier, idsToString} import scorex.core.network.NetworkController.ReceivableMessages.{PenalizePeer, SendToNetwork} import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages._ import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoStateReader} @@ -26,7 +26,6 @@ import scorex.core.network._ import scorex.core.network.message.{InvData, Message, ModifiersData} import scorex.core.serialization.ScorexSerializer import scorex.core.settings.NetworkSettings -import scorex.core.transaction.Transaction import scorex.core.utils.{NetworkTimeProvider, ScorexEncoding} import scorex.core.validation.MalformedModifierError import scorex.util.{ModifierId, ScorexLogging} @@ -257,7 +256,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, context.system.scheduler.scheduleAtFixedRate(healthCheckDelay, healthCheckRate, viewHolderRef, IsChainHealthy)(ex, self) } - protected def broadcastModifierInv(modTypeId: ModifierTypeId, modId: ModifierId): Unit = { + protected def broadcastModifierInv(modTypeId: NetworkObjectTypeId.Value, modId: ModifierId): Unit = { val msg = Message(InvSpec, Right(InvData(modTypeId, Seq(modId))), None) networkControllerRef ! SendToNetwork(msg, Broadcast) } @@ -271,7 +270,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * (in history database available via `historyReader` interface, or delivery tracker cache, thus * downloading of the modifier is needed. */ - private def downloadRequired(historyReader: ErgoHistory)(modifierTypeId: ModifierTypeId, id: ModifierId): Boolean = { + private def downloadRequired(historyReader: ErgoHistory)(modifierTypeId: NetworkObjectTypeId.Value, id: ModifierId): Boolean = { deliveryTracker.status(id, modifierTypeId, Array(historyReader)) == ModifiersStatus.Unknown } @@ -341,7 +340,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, // Send history extension to the (less developed) peer 'remote' which does not have it. def sendExtension(remote: ConnectedPeer, - ext: Seq[(ModifierTypeId, ModifierId)]): Unit = { + ext: Seq[(NetworkObjectTypeId.Value, ModifierId)]): Unit = { ext.groupBy(_._1).mapValues(_.map(_._2)).foreach { case (mid, mods) => networkControllerRef ! SendToNetwork(Message(InvSpec, Right(InvData(mid, mods)), None), SendToPeer(remote)) @@ -526,7 +525,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * (non-zero if we're re-requesting the block section, in this case, there should be only * one id to request in `modifierIds` */ - def requestBlockSection(modifierTypeId: ModifierTypeId, + def requestBlockSection(modifierTypeId: NetworkObjectTypeId.Value, modifierIds: Seq[ModifierId], peer: ConnectedPeer, checksDone: Int = 0): Unit = { @@ -546,7 +545,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } def onDownloadRequest(historyReader: ErgoHistory): Receive = { - case DownloadRequest(modifiersToFetch: Map[ModifierTypeId, Seq[ModifierId]]) => + case DownloadRequest(modifiersToFetch: Map[NetworkObjectTypeId.Value, Seq[ModifierId]]) => log.debug(s"Downloading via DownloadRequest: $modifiersToFetch") if(modifiersToFetch.nonEmpty) { requestDownload( @@ -581,7 +580,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, */ protected def requestDownload(maxModifiers: Int, minModifiersPerBucket: Int, maxModifiersPerBucket: Int) (getPeersOpt: => Option[Iterable[ConnectedPeer]]) - (fetchMax: Int => Map[ModifierTypeId, Seq[ModifierId]]): Unit = + (fetchMax: Int => Map[NetworkObjectTypeId.Value, Seq[ModifierId]]): Unit = getPeersOpt .foreach { peers => val modifiersByBucket = ElementPartitioner.distribute(peers, maxModifiers, minModifiersPerBucket, maxModifiersPerBucket)(fetchMax) @@ -624,7 +623,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } private def blockSectionsFromRemote(hr: ErgoHistory, - typeId: ModifierTypeId, + typeId: NetworkObjectTypeId.Value, requestedModifiers: Map[ModifierId, Array[Byte]], remote: ConnectedPeer): Unit = { Constants.modifierSerializers.get(typeId) match { @@ -674,7 +673,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, // filter out non-requested modifiers val requestedModifiers = processSpam(remote, typeId, modifiers, blockAppliedTxsCache) - if (typeId == Transaction.ModifierTypeId) { + if (typeId == ErgoTransaction.modifierTypeId) { transactionsFromRemote(requestedModifiers, mp, remote) } else { blockSectionsFromRemote(hr, typeId, requestedModifiers, remote) @@ -687,7 +686,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, */ def parseAndProcessTransaction(id: ModifierId, bytes: Array[Byte], remote: ConnectedPeer): Unit = { if (bytes.length > settings.nodeSettings.maxTransactionSize) { - deliveryTracker.setInvalid(id, Transaction.ModifierTypeId) + deliveryTracker.setInvalid(id, ErgoTransaction.modifierTypeId) penalizeMisbehavingPeer(remote) log.warn(s"Transaction size ${bytes.length} from ${remote.toString} " + s"exceeds limit ${settings.nodeSettings.maxTransactionSize}") @@ -711,7 +710,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * @return collection of parsed modifiers */ def parseModifiers[M <: NodeViewModifier](modifiers: Map[ModifierId, Array[Byte]], - modifierTypeId: ModifierTypeId, + modifierTypeId: NetworkObjectTypeId.Value, serializer: ScorexSerializer[M], remote: ConnectedPeer): Iterable[M] = { modifiers.flatMap { case (id, bytes) => @@ -736,7 +735,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * @return ids and bytes of modifiers that were requested by our node */ def processSpam(remote: ConnectedPeer, - typeId: ModifierTypeId, + typeId: NetworkObjectTypeId.Value, modifiers: Map[ModifierId, Array[Byte]], blockAppliedTxsCache: FixedSizeApproximateCacheQueue): Map[ModifierId, Array[Byte]] = { val modifiersByStatus = @@ -747,7 +746,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, val spam = modifiersByStatus.filterKeys(_ != Requested) if (spam.nonEmpty) { - if (typeId == Transaction.ModifierTypeId) { + if (typeId == ErgoTransaction.modifierTypeId) { // penalize a peer for sending TXs that have been already applied to a block val spammyTxs = modifiers.filterKeys(blockAppliedTxsCache.mightContain) if (spammyTxs.nonEmpty) { @@ -792,7 +791,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, val modifierTypeId = invData.typeId val newModifierIds = modifierTypeId match { - case Transaction.ModifierTypeId => + case ErgoTransaction.modifierTypeId => if (txAcceptanceFilter) { val unknownMods = { @@ -847,11 +846,11 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, //other node asking for objects by their ids protected def modifiersReq(hr: ErgoHistory, mp: ErgoMemPool, invData: InvData, remote: ConnectedPeer): Unit = { val objs: Seq[(ModifierId, Array[Byte])] = invData.typeId match { - case typeId: ModifierTypeId if typeId == Transaction.ModifierTypeId => + case typeId: NetworkObjectTypeId.Value if typeId == ErgoTransaction.modifierTypeId => mp.getAll(invData.ids).map { unconfirmedTx => unconfirmedTx.transaction.id -> unconfirmedTx.transactionBytes.getOrElse(unconfirmedTx.transaction.bytes) } - case expectedTypeId: ModifierTypeId => + case expectedTypeId: NetworkObjectTypeId.Value => invData.ids.flatMap { id => hr.modifierTypeAndBytesById(id).flatMap { case (mTypeId, bytes) => if (mTypeId == expectedTypeId) { @@ -920,7 +919,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, if (deliveryTracker.status(modifierId, modifierTypeId, Seq.empty) == ModifiersStatus.Requested) { // If transaction not delivered on time, we just forget about it. // It could be removed from other peer's mempool, so no reason to penalize the peer. - if (modifierTypeId == Transaction.ModifierTypeId) { + if (modifierTypeId == ErgoTransaction.modifierTypeId) { deliveryTracker.clearStatusForModifier(modifierId, modifierTypeId, ModifiersStatus.Requested) } else { // A block section is not delivered on time. @@ -1049,7 +1048,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, case st@SuccessfulTransaction(utx) => val tx = utx.transaction - deliveryTracker.setHeld(tx.id, Transaction.ModifierTypeId) + deliveryTracker.setHeld(tx.id, ErgoTransaction.modifierTypeId) processMempoolResult(st) broadcastModifierInv(tx) @@ -1228,7 +1227,7 @@ object ErgoNodeViewSynchronizer { // getLocalSyncInfo messages case object SendLocalSyncInfo - case class ResponseFromLocal(source: ConnectedPeer, modifierTypeId: ModifierTypeId, localObjects: Seq[(ModifierId, Array[Byte])]) + case class ResponseFromLocal(source: ConnectedPeer, modifierTypeId: NetworkObjectTypeId, localObjects: Seq[(ModifierId, Array[Byte])]) /** * Check delivery of modifier with type `modifierTypeId` and id `modifierId`. @@ -1237,7 +1236,7 @@ object ErgoNodeViewSynchronizer { * */ case class CheckDelivery(source: ConnectedPeer, - modifierTypeId: ModifierTypeId, + modifierTypeId: NetworkObjectTypeId.Value, modifierId: ModifierId) trait PeerManagerEvent @@ -1284,19 +1283,19 @@ object ErgoNodeViewSynchronizer { */ case class FailedOnRecheckTransaction(id : ModifierId, error: Throwable) extends ModificationOutcome - case class RecoverableFailedModification(typeId: ModifierTypeId, modifierId: ModifierId, error: Throwable) extends ModificationOutcome + case class RecoverableFailedModification(typeId: NetworkObjectTypeId.Value, modifierId: ModifierId, error: Throwable) extends ModificationOutcome - case class SyntacticallyFailedModification(typeId: ModifierTypeId, modifierId: ModifierId, error: Throwable) extends ModificationOutcome + case class SyntacticallyFailedModification(typeId: NetworkObjectTypeId.Value, modifierId: ModifierId, error: Throwable) extends ModificationOutcome /** * Signal associated with stateful validation of a block section */ - case class SemanticallyFailedModification(typeId: ModifierTypeId, modifierId: ModifierId, error: Throwable) extends ModificationOutcome + case class SemanticallyFailedModification(typeId: NetworkObjectTypeId.Value, modifierId: ModifierId, error: Throwable) extends ModificationOutcome /** * Signal associated with stateless validation of a block section */ - case class SyntacticallySuccessfulModifier(typeId: ModifierTypeId, modifierId: ModifierId) extends ModificationOutcome + case class SyntacticallySuccessfulModifier(typeId: NetworkObjectTypeId.Value, modifierId: ModifierId) extends ModificationOutcome /** * Signal sent by node view holder when a full block is applied to state @@ -1312,7 +1311,7 @@ object ErgoNodeViewSynchronizer { */ case class BlockSectionsProcessingCacheUpdate(headersCacheSize: Int, blockSectionsCacheSize: Int, - cleared: (ModifierTypeId, Seq[ModifierId])) + cleared: (NetworkObjectTypeId.Value, Seq[ModifierId])) /** * Command to re-check mempool to clean transactions become invalid while sitting in the mempool up diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index 326f63052f..954566ccef 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -4,10 +4,9 @@ import akka.actor.SupervisorStrategy.Escalate import akka.actor.{Actor, ActorRef, ActorSystem, OneForOneStrategy, Props} import org.ergoplatform.ErgoApp import org.ergoplatform.ErgoApp.CriticalSystemException -import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} -import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NetworkObjectTypeId} import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.mempool.ErgoMemPool import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome @@ -26,7 +25,9 @@ import scorex.core.utils.{NetworkTimeProvider, ScorexEncoding} import scorex.core.validation.RecoverableModifierError import scorex.util.{ModifierId, ScorexLogging} import spire.syntax.all.cfor + import java.io.File +import org.ergoplatform.modifiers.history.extension.Extension import scala.annotation.tailrec import scala.collection.mutable @@ -35,8 +36,9 @@ import scala.util.{Failure, Success, Try} /** * Composite local view of the node * - * Contains instances for History, ErgoState, Wallet, MemoryPool. - * Updates them atomically. + * Contains instances for History, ErgoState, Vault, MemoryPool. + * The instances are read-only for external world. + * Updates of the composite view instances are to be performed atomically. * */ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSettings, @@ -135,7 +137,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti //TODO: actually, pi.toDownload contains only 1 modifierid per type, //TODO: see the only case where toDownload is not empty during ProgressInfo construction //TODO: so the code below can be optimized - val toDownload = mutable.Map[ModifierTypeId, Seq[ModifierId]]() + val toDownload = mutable.Map[NetworkObjectTypeId.Value, Seq[ModifierId]]() pi.toDownload.foreach { case (tid, mid) => toDownload.put(tid, toDownload.getOrElse(tid, Seq()) :+ mid) } @@ -143,7 +145,6 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti } - private def trimChainSuffix(suffix: IndexedSeq[BlockSection], rollbackPoint: ModifierId): IndexedSeq[BlockSection] = { val idx = suffix.indexWhere(_.id == rollbackPoint) @@ -235,7 +236,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti updateInfo.state.applyModifier(modToApply, chainTipOpt)(lm => pmodModify(lm.pmod, local = true)) match { case Success(stateAfterApply) => history.reportModifierIsValid(modToApply).map { newHis => - if(modToApply.modifierTypeId == ErgoFullBlock.modifierTypeId) { + if (modToApply.modifierTypeId == ErgoFullBlock.modifierTypeId) { context.system.eventStream.publish(FullBlockApplied(modToApply.asInstanceOf[ErgoFullBlock].header)) } UpdateInformation(newHis, stateAfterApply, None, None, updateInfo.suffix :+ modToApply) @@ -342,7 +343,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti val at0 = System.currentTimeMillis() applyFromCacheLoop(modifiersCache) val at = System.currentTimeMillis() - log.debug(s"Application time: ${at-at0}") + log.debug(s"Application time: ${at - at0}") val cleared = modifiersCache.cleanOverfull() @@ -424,10 +425,12 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti } //todo: update state in async way? + /** * Remote and local persistent modifiers need to be appended to history, applied to state * which also needs to be git propagated to mempool and wallet - * @param pmod Remote or local persistent modifier + * + * @param pmod Remote or local persistent modifier * @param local whether the modifier was generated locally or not */ protected def pmodModify(pmod: BlockSection, local: Boolean): Unit = { @@ -462,7 +465,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti val fullBlockHeight = newHistory.fullBlockHeight val almostSynced = (headersHeight - fullBlockHeight) < almostSyncedGap - val newMemPool = if(almostSynced) { + val newMemPool = if (almostSynced) { updateMemPool(progressInfo.toRemove, blocksApplied, memoryPool()) } else { memoryPool() @@ -717,12 +720,13 @@ object ErgoNodeViewHolder { case class BlockAppliedTransactions(txs: Seq[ModifierId]) extends NodeViewHolderEvent - case class DownloadRequest(modifiersToFetch: Map[ModifierTypeId, Seq[ModifierId]]) extends NodeViewHolderEvent + case class DownloadRequest(modifiersToFetch: Map[NetworkObjectTypeId.Value, Seq[ModifierId]]) extends NodeViewHolderEvent case class CurrentView[State](history: ErgoHistory, state: State, vault: ErgoWallet, pool: ErgoMemPool) /** * Checks whether chain got stuck by comparing timestamp of bestFullBlock or last time a modifier was applied to history. + * * @param progress metadata of last chain update * @return ChainIsHealthy if chain is healthy and ChainIsStuck(error) with details if it got stuck */ diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala index ca5352f300..e4087216ad 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala @@ -4,12 +4,12 @@ import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.{Header, PreGenesisHeader} import org.ergoplatform.modifiers.history.popow.{NipopowAlgos, NipopowProof, PoPowHeader, PoPowParams} -import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NonHeaderBlockSection} +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NetworkObjectTypeId, NonHeaderBlockSection} import org.ergoplatform.nodeView.history.ErgoHistory.Height import org.ergoplatform.nodeView.history.storage._ import org.ergoplatform.nodeView.history.storage.modifierprocessors._ import org.ergoplatform.settings.ErgoSettings -import scorex.core.{ModifierTypeId, NodeViewComponent} +import scorex.core.NodeViewComponent import scorex.core.consensus.{ContainsModifiers, Equal, Fork, ModifierSemanticValidity, Older, PeerChainStatus, Unknown, Younger} import scorex.core.utils.ScorexEncoding import scorex.core.validation.MalformedModifierError @@ -30,7 +30,7 @@ trait ErgoHistoryReader with ScorexLogging with ScorexEncoding { - type ModifierIds = Seq[(ModifierTypeId, ModifierId)] + type ModifierIds = Seq[(NetworkObjectTypeId.Value, ModifierId)] protected[history] val historyStorage: HistoryStorage @@ -73,7 +73,7 @@ trait ErgoHistoryReader * @param id - modifier id * @return type and raw bytes of semantically valid ErgoPersistentModifier with the given id it is in history */ - def modifierTypeAndBytesById(id: ModifierId): Option[(ModifierTypeId, Array[Byte])] = + def modifierTypeAndBytesById(id: ModifierId): Option[(NetworkObjectTypeId.Value, Array[Byte])] = if (isSemanticallyValid(id) != ModifierSemanticValidity.Invalid) { historyStorage.modifierTypeAndBytesById(id) } else { diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index b139f61db9..1080ae5642 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -1,16 +1,13 @@ package org.ergoplatform.nodeView.history.storage import com.github.benmanes.caffeine.cache.Caffeine -import org.ergoplatform.modifiers.BlockSection +import org.ergoplatform.modifiers.{BlockSection, NetworkObjectTypeId} import org.ergoplatform.modifiers.history.HistoryModifierSerializer import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.settings.{Algos, CacheSettings, ErgoSettings} -import scorex.core.ModifierTypeId import scorex.core.utils.ScorexEncoding import scorex.db.{ByteArrayWrapper, LDBFactory, LDBKVStore} import scorex.util.{ModifierId, ScorexLogging, idToBytes} -import supertagged.PostfixSugar - import scala.util.{Failure, Success, Try} /** @@ -58,8 +55,8 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c objectsStore.get(idToBytes(id)).map(_.tail) // removing modifier type byte with .tail } - def modifierTypeAndBytesById(id: ModifierId): Option[(ModifierTypeId, Array[Byte])] = { - objectsStore.get(idToBytes(id)).map(bs => (bs.head @@ ModifierTypeId, bs.tail)) // first byte is type id, tail is modifier bytes + def modifierTypeAndBytesById(id: ModifierId): Option[(NetworkObjectTypeId.Value, Array[Byte])] = { + objectsStore.get(idToBytes(id)).map(bs => (NetworkObjectTypeId.fromByte(bs.head), bs.tail)) // first byte is type id, tail is modifier bytes } def modifierById(id: ModifierId): Option[BlockSection] = diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala index 97af3cc922..7d5f047141 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala @@ -1,10 +1,9 @@ package org.ergoplatform.nodeView.history.storage.modifierprocessors -import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.modifiers.{ErgoFullBlock, NetworkObjectTypeId} import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.settings.{ChainSettings, ErgoSettings, NodeConfigurationSettings} -import scorex.core.ModifierTypeId import scorex.core.utils.NetworkTimeProvider import scorex.util.{ModifierId, ScorexLogging} @@ -42,12 +41,12 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { /** * Get modifier ids to download to synchronize full blocks * @param howManyPerType how many ModifierIds per ModifierTypeId to fetch - * @param condition filter only ModifierIds that pass this condition + * @param condition only ModifierIds which pass filter are included into results * @return next max howManyPerType ModifierIds by ModifierTypeId to download filtered by condition */ def nextModifiersToDownload(howManyPerType: Int, estimatedTip: Option[Int], - condition: (ModifierTypeId, ModifierId) => Boolean): Map[ModifierTypeId, Seq[ModifierId]] = { + condition: (NetworkObjectTypeId.Value, ModifierId) => Boolean): Map[NetworkObjectTypeId.Value, Seq[ModifierId]] = { val FullBlocksToDownloadAhead = 192 // how many full blocks to download forwards during active sync @@ -56,8 +55,8 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { @tailrec def continuation(height: Int, - acc: Map[ModifierTypeId, Vector[ModifierId]], - maxHeight: Int = Int.MaxValue): Map[ModifierTypeId, Vector[ModifierId]] = { + acc: Map[NetworkObjectTypeId.Value, Vector[ModifierId]], + maxHeight: Int = Int.MaxValue): Map[NetworkObjectTypeId.Value, Vector[ModifierId]] = { if (height > maxHeight) { acc } else { @@ -81,7 +80,7 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { bestFullBlockOpt match { case _ if !isHeadersChainSynced || !nodeSettings.verifyTransactions => - // do not download full blocks if no headers-chain synced yet and suffix enabled or SPV mode + // do not download full blocks if no headers-chain synced yet or SPV mode Map.empty case Some(fb) if farAwayFromBeingSynced(fb) => // when far away from blockchain tip @@ -100,7 +99,7 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { /** * Checks whether it's time to download full chain, and returns toDownload modifiers */ - protected def toDownload(header: Header): Seq[(ModifierTypeId, ModifierId)] = { + protected def toDownload(header: Header): Seq[(NetworkObjectTypeId.Value, ModifierId)] = { if (!nodeSettings.verifyTransactions) { // A regime that do not download and verify transaction Nil @@ -117,7 +116,7 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { } } - def requiredModifiersForHeader(h: Header): Seq[(ModifierTypeId, ModifierId)] = { + def requiredModifiersForHeader(h: Header): Seq[(NetworkObjectTypeId.Value, ModifierId)] = { if (!nodeSettings.verifyTransactions) { Nil } else if (nodeSettings.stateType.requireProofs) { diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala index b585c0bada..b0a5e6d7ee 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala @@ -13,7 +13,6 @@ import org.ergoplatform.settings.{Algos, Parameters} import org.ergoplatform.utils.LoggingUtil import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import scorex.core._ -import scorex.core.transaction.Transaction import scorex.core.transaction.state.TransactionValidation import scorex.core.utils.ScorexEncoding import scorex.core.validation.{ModifierValidator} @@ -101,7 +100,7 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 opsResult } ModifierValidator(stateContext.validationSettings) - .validateNoFailure(fbOperationFailed, blockOpsTry, Transaction.ModifierTypeId) + .validateNoFailure(fbOperationFailed, blockOpsTry, ErgoTransaction.modifierTypeId) .validateEquals(fbDigestIncorrect, expectedDigest, persistentProver.digest, headerId, Header.modifierTypeId) .result .toTry diff --git a/src/main/scala/org/ergoplatform/settings/Constants.scala b/src/main/scala/org/ergoplatform/settings/Constants.scala index b5cece06e5..c258107856 100644 --- a/src/main/scala/org/ergoplatform/settings/Constants.scala +++ b/src/main/scala/org/ergoplatform/settings/Constants.scala @@ -1,14 +1,14 @@ package org.ergoplatform.settings import org.ergoplatform.mining.difficulty.RequiredDifficulty +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionSerializer} import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} -import org.ergoplatform.modifiers.mempool.ErgoTransactionSerializer +import org.ergoplatform.modifiers.mempool.{ErgoTransaction, ErgoTransactionSerializer} import org.ergoplatform.nodeView.history.ErgoHistory.Difficulty import scorex.core.serialization.ScorexSerializer -import scorex.core.transaction.Transaction -import scorex.core.{ModifierTypeId, NodeViewModifier} +import scorex.core.NodeViewModifier import sigmastate.Values import sigmastate.Values.ErgoTree @@ -44,12 +44,12 @@ object Constants { // Number of last block headers available is scripts from ErgoStateContext val LastHeadersInContext = 10 - val modifierSerializers: Map[ModifierTypeId, ScorexSerializer[_ <: NodeViewModifier]] = + val modifierSerializers: Map[NetworkObjectTypeId.Value, ScorexSerializer[_ <: NodeViewModifier]] = Map(Header.modifierTypeId -> HeaderSerializer, Extension.modifierTypeId -> ExtensionSerializer, BlockTransactions.modifierTypeId -> BlockTransactionsSerializer, ADProofs.modifierTypeId -> ADProofsSerializer, - Transaction.ModifierTypeId -> ErgoTransactionSerializer) + ErgoTransaction.modifierTypeId -> ErgoTransactionSerializer) val SoftForkEpochs = 32 //about 45.5 days diff --git a/src/main/scala/org/ergoplatform/settings/ValidationRules.scala b/src/main/scala/org/ergoplatform/settings/ValidationRules.scala index 2be07be4c3..49d516c65f 100644 --- a/src/main/scala/org/ergoplatform/settings/ValidationRules.scala +++ b/src/main/scala/org/ergoplatform/settings/ValidationRules.scala @@ -1,14 +1,13 @@ package org.ergoplatform.settings import org.ergoplatform.SigmaConstants.{MaxBoxSize, MaxPropositionBytes} -import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.modifiers.{ErgoFullBlock, NetworkObjectTypeId} import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.history.{ADProofs, BlockTransactions} import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.wallet.boxes.ErgoBoxAssetExtractor -import scorex.core.ModifierTypeId import scorex.core.validation.{InvalidModifier, ModifierValidator} import scorex.core.validation.ValidationResult.Invalid import scorex.util.ModifierId @@ -306,7 +305,7 @@ object ValidationRules { val fbDigestIncorrect: Short = 501 - def errorMessage(id: Short, details: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): String = { + def errorMessage(id: Short, details: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): String = { ValidationRules.rulesSpec(id) .invalidMod(InvalidModifier(details, modifierId, modifierTypeId)) .errors @@ -314,10 +313,10 @@ object ValidationRules { .message } - private def recoverable(errorMessage: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + private def recoverable(errorMessage: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = ModifierValidator.error(errorMessage, modifierId, modifierTypeId) - private def fatal(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + private def fatal(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = ModifierValidator.fatal(error, modifierId, modifierTypeId) } diff --git a/src/main/scala/org/ergoplatform/tools/ValidationRulesPrinter.scala b/src/main/scala/org/ergoplatform/tools/ValidationRulesPrinter.scala index b78453f0b1..ed668f4904 100644 --- a/src/main/scala/org/ergoplatform/tools/ValidationRulesPrinter.scala +++ b/src/main/scala/org/ergoplatform/tools/ValidationRulesPrinter.scala @@ -1,7 +1,7 @@ package org.ergoplatform.tools +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.settings.ValidationRules -import scorex.core.ModifierTypeId import scorex.core.validation.InvalidModifier import scorex.util.{ModifierId, ScorexLogging, bytesToId} @@ -14,7 +14,7 @@ object ValidationRulesPrinter extends App with ScorexLogging { printHeader() rules.toSeq.sortBy(_._1).foreach { r => - val rule = r._2.invalidMod(InvalidModifier("", emptyModifierId, ModifierTypeId @@ 0.toByte)).errors.head.message.trim + val rule = r._2.invalidMod(InvalidModifier("", emptyModifierId, NetworkObjectTypeId.fromByte(0))).errors.head.message.trim val activated = r._2.isActive val mayBeDisabled = r._2.mayBeDisabled val modifiers = r._2.affectedClasses.map(_.getSimpleName).mkString(", ") @@ -37,7 +37,7 @@ object ValidationRulesPrinter extends App with ScorexLogging { printHeader() } - if (r._2.invalidMod(InvalidModifier("", emptyModifierId, ModifierTypeId @@ 0.toByte)).isFatal) { + if (r._2.invalidMod(InvalidModifier("", emptyModifierId, NetworkObjectTypeId.fromByte(0))).isFatal) { // we only mention fatal errors here println(s" ${r._1} & $rule & ${boolToLatex(mayBeDisabled)} & ${boolToLatex(activated)} & $modifiers \\\\") diff --git a/src/main/scala/scorex/core/NodeViewModifier.scala b/src/main/scala/scorex/core/NodeViewModifier.scala index b2ce44fdb7..8e3be7e142 100644 --- a/src/main/scala/scorex/core/NodeViewModifier.scala +++ b/src/main/scala/scorex/core/NodeViewModifier.scala @@ -1,24 +1,24 @@ package scorex.core +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.modifiers.mempool.ErgoTransaction import scorex.core.serialization.BytesSerializable import scorex.core.utils.ScorexEncoding - sealed trait NodeViewModifier extends BytesSerializable with ScorexEncoding {self => - val modifierTypeId: ModifierTypeId + val modifierTypeId: NetworkObjectTypeId.Value //todo: check statically or dynamically output size def id: scorex.util.ModifierId - def encodedId: String = encoder.encodeId(id) - override def equals(obj: scala.Any): Boolean = obj match { case that: NodeViewModifier => (that.id == id) && (that.modifierTypeId == modifierTypeId) case _ => false } + def encodedId: String = id + } trait EphemerealNodeViewModifier extends NodeViewModifier diff --git a/src/main/scala/scorex/core/consensus/ProgressInfo.scala b/src/main/scala/scorex/core/consensus/ProgressInfo.scala index d540d897ef..c292b3760f 100644 --- a/src/main/scala/scorex/core/consensus/ProgressInfo.scala +++ b/src/main/scala/scorex/core/consensus/ProgressInfo.scala @@ -1,7 +1,8 @@ package scorex.core.consensus +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.core.utils.ScorexEncoder -import scorex.core.{ModifierTypeId, PersistentNodeViewModifier} +import scorex.core.PersistentNodeViewModifier import scorex.util.ModifierId /** @@ -16,7 +17,7 @@ import scorex.util.ModifierId case class ProgressInfo[PM <: PersistentNodeViewModifier](branchPoint: Option[ModifierId], toRemove: Seq[PM], toApply: Seq[PM], - toDownload: Seq[(ModifierTypeId, ModifierId)]) + toDownload: Seq[(NetworkObjectTypeId.Value, ModifierId)]) (implicit encoder: ScorexEncoder) { if (toRemove.nonEmpty) diff --git a/src/main/scala/scorex/core/core.scala b/src/main/scala/scorex/core/core.scala index bb02ad8669..8609bd0eb1 100644 --- a/src/main/scala/scorex/core/core.scala +++ b/src/main/scala/scorex/core/core.scala @@ -1,5 +1,6 @@ package scorex +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.core.network.message.InvData import scorex.core.utils.ScorexEncoder import scorex.util.encode.Base16 @@ -7,23 +8,18 @@ import supertagged.TaggedType package object core { - //TODO implement ModifierTypeId as a trait - object ModifierTypeId extends TaggedType[Byte] - object VersionTag extends TaggedType[String] - type ModifierTypeId = ModifierTypeId.Type - type VersionTag = VersionTag.Type - def idsToString(ids: Seq[(ModifierTypeId, util.ModifierId)])(implicit enc: ScorexEncoder): String = { + def idsToString(ids: Seq[(NetworkObjectTypeId.Value, util.ModifierId)])(implicit enc: ScorexEncoder): String = { List(ids.headOption, ids.lastOption) .flatten .map { case (typeId, id) => s"($typeId,${enc.encodeId(id)})" } .mkString("[", "..", "]") } - def idsToString(modifierType: ModifierTypeId, ids: Seq[util.ModifierId])(implicit encoder: ScorexEncoder): String = { + def idsToString(modifierType: NetworkObjectTypeId.Value, ids: Seq[util.ModifierId])(implicit encoder: ScorexEncoder): String = { idsToString(ids.map(id => (modifierType, id))) } diff --git a/src/main/scala/scorex/core/network/DeliveryTracker.scala b/src/main/scala/scorex/core/network/DeliveryTracker.scala index 283d3ce622..b6231445a5 100644 --- a/src/main/scala/scorex/core/network/DeliveryTracker.scala +++ b/src/main/scala/scorex/core/network/DeliveryTracker.scala @@ -2,11 +2,11 @@ package scorex.core.network import akka.actor.Cancellable import io.circe.{Encoder, Json} +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.CheckDelivery import org.ergoplatform.nodeView.mempool.ExpiringApproximateCache import org.ergoplatform.settings.{ErgoSettings, NetworkCacheSettings} -import scorex.core.ModifierTypeId import scorex.core.consensus.ContainsModifiers import scorex.core.network.DeliveryTracker._ import scorex.core.network.ModifiersStatus._ @@ -45,10 +45,10 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, desiredSizeOfExpectingModifierQueue: Int) extends ScorexLogging with ScorexEncoding { // when a remote peer is asked for a modifier we add the requested data to `requested` - protected val requested: mutable.Map[ModifierTypeId, Map[ModifierId, RequestedInfo]] = mutable.Map() + protected val requested: mutable.Map[NetworkObjectTypeId.Value, Map[ModifierId, RequestedInfo]] = mutable.Map() // when our node received a modifier we put it to `received` - protected val received: mutable.Map[ModifierTypeId, Map[ModifierId, ConnectedPeer]] = mutable.Map() + protected val received: mutable.Map[NetworkObjectTypeId.Value, Map[ModifierId, ConnectedPeer]] = mutable.Map() private val desiredSizeOfExpectingHeaderQueue: Int = desiredSizeOfExpectingModifierQueue * 8 @@ -95,7 +95,9 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, * Since this class do not keep statuses for modifiers that are already in NodeViewHolder, * `modifierKeepers` are required here to check that modifier is in `Held` status */ - def status(modifierId: ModifierId, modifierTypeId: ModifierTypeId, modifierKeepers: Seq[ContainsModifiers[_]]): ModifiersStatus = + def status(modifierId: ModifierId, + modifierTypeId: NetworkObjectTypeId.Value, + modifierKeepers: Seq[ContainsModifiers[_]]): ModifiersStatus = if (received.get(modifierTypeId).exists(_.contains(modifierId))) Received else if (requested.get(modifierTypeId).exists(_.contains(modifierId))) Requested else if (invalidModifierCache.mightContain(modifierId)) Invalid @@ -112,7 +114,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, /** * Set status of modifier with id `id` to `Requested` */ - def setRequested(typeId: ModifierTypeId, + def setRequested(typeId: NetworkObjectTypeId.Value, id: ModifierId, supplier: ConnectedPeer, checksDone: Int = 0) @@ -128,12 +130,12 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, /** * @return - information on requested modifier delivery tracker potentially has */ - def getRequestedInfo(typeId: ModifierTypeId, id: ModifierId): Option[RequestedInfo] = { + def getRequestedInfo(typeId: NetworkObjectTypeId.Value, id: ModifierId): Option[RequestedInfo] = { requested.get(typeId).flatMap(_.get(id)) } /** Get peer we're communicating with in regards with modifier `id` **/ - def getSource(id: ModifierId, modifierTypeId: ModifierTypeId): Option[ConnectedPeer] = { + def getSource(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Option[ConnectedPeer] = { status(id, modifierTypeId, Seq.empty) match { case Requested => requested.get(modifierTypeId).flatMap(_.get(id)).map(_.peer) case Received => received.get(modifierTypeId).flatMap(_.get(id)) @@ -145,7 +147,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, * Modified with id `id` is permanently invalid - set its status to `Invalid` * and return [[ConnectedPeer]] which sent bad modifier. */ - def setInvalid(id: ModifierId, modifierTypeId: ModifierTypeId): Option[ConnectedPeer] = { + def setInvalid(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Option[ConnectedPeer] = { val oldStatus: ModifiersStatus = status(id, modifierTypeId, Seq.empty) val transitionCheck = tryWithLogging { checkStatusTransition(oldStatus, Invalid) @@ -188,7 +190,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, /** * Modifier with id `id` was successfully applied to history - set its status to `Held`. */ - def setHeld(id: ModifierId, modifierTypeId: ModifierTypeId): Unit = + def setHeld(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Unit = tryWithLogging { val oldStatus = status(id, modifierTypeId, Seq.empty) checkStatusTransition(oldStatus, Held) @@ -203,7 +205,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, * this modifier was removed from cache because cache is overfull or * we stop trying to download this modifiers due to exceeded number of retries */ - def setUnknown(id: ModifierId, modifierTypeId: ModifierTypeId): Unit = + def setUnknown(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Unit = tryWithLogging { val oldStatus = status(id, modifierTypeId, Seq.empty) checkStatusTransition(oldStatus, Unknown) @@ -213,7 +215,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, /** * Modifier with id `id` was received from remote peer - set its status to `Received`. */ - def setReceived(id: ModifierId, modifierTypeId: ModifierTypeId, sender: ConnectedPeer): Unit = + def setReceived(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value, sender: ConnectedPeer): Unit = tryWithLogging { val oldStatus = status(id, modifierTypeId, Seq.empty) checkStatusTransition(oldStatus, Received) @@ -251,7 +253,7 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, case _ => false } - def clearStatusForModifier(id: ModifierId, modifierTypeId: ModifierTypeId, oldStatus: ModifiersStatus): Unit = + def clearStatusForModifier(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value, oldStatus: ModifiersStatus): Unit = oldStatus match { case Requested => requested.flatAdjust(modifierTypeId)(_.map { infoById => @@ -325,16 +327,16 @@ object DeliveryTracker { } case class FullInfo( - invalidModifierApproxSize: Long, - requested: Seq[(ModifierTypeId, Map[ModifierId, RequestedInfo])], - received: Seq[(ModifierTypeId, Map[ModifierId, ConnectedPeer])] + invalidModifierApproxSize: Long, + requested: Seq[(NetworkObjectTypeId.Value, Map[ModifierId, RequestedInfo])], + received: Seq[(NetworkObjectTypeId.Value, Map[ModifierId, ConnectedPeer])] ) object FullInfo { import io.circe.syntax._ implicit val encodeState: Encoder[FullInfo] = new Encoder[FullInfo] { - def nestedMapAsJson[T : Encoder](requested: Seq[(ModifierTypeId, Map[ModifierId, T])]): Json = + def nestedMapAsJson[T : Encoder](requested: Seq[(NetworkObjectTypeId.Value, Map[ModifierId, T])]): Json = Json.obj( requested.map { case (k, v) => k.toString -> Json.obj(v.mapValues(_.asJson).toSeq:_*) diff --git a/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala b/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala index 7e09f33756..68e5f0364f 100644 --- a/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala +++ b/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala @@ -1,20 +1,21 @@ package scorex.core.network.message +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.core.consensus.SyncInfo import scorex.core.network._ import scorex.core.network.message.Message.MessageCode import scorex.core.serialization.ScorexSerializer -import scorex.core.{ModifierTypeId, NodeViewModifier} +import scorex.core.NodeViewModifier import scorex.util.Extensions._ import scorex.util.serialization.{Reader, Writer} import scorex.util.{ModifierId, ScorexLogging, bytesToId, idToBytes} import scala.collection.immutable -case class ModifiersData(typeId: ModifierTypeId, modifiers: Map[ModifierId, Array[Byte]]) +case class ModifiersData(typeId: NetworkObjectTypeId.Value, modifiers: Map[ModifierId, Array[Byte]]) -case class InvData(typeId: ModifierTypeId, ids: Seq[ModifierId]) +case class InvData(typeId: NetworkObjectTypeId.Value, ids: Seq[ModifierId]) /** * The `SyncInfo` message requests an `Inv` message that provides modifier ids @@ -63,7 +64,7 @@ object InvSpec extends MessageSpecV1[InvData] { } override def parse(r: Reader): InvData = { - val typeId = ModifierTypeId @@ r.getByte() + val typeId = NetworkObjectTypeId.fromByte(r.getByte()) val count = r.getUInt().toIntExact require(count > 0, "empty inv list") require(count <= maxInvObjects, s"$count elements in a message while limit is $maxInvObjects") @@ -142,7 +143,7 @@ object ModifiersSpec extends MessageSpecV1[ModifiersData] with ScorexLogging { } override def parse(r: Reader): ModifiersData = { - val typeId = ModifierTypeId @@ r.getByte() // 1 byte + val typeId = NetworkObjectTypeId.fromByte(r.getByte()) // 1 byte val count = r.getUInt().toIntExact // 8 bytes require(count > 0, s"Illegal message with 0 modifiers of type $typeId") val resMap = immutable.Map.newBuilder[ModifierId, Array[Byte]] diff --git a/src/main/scala/scorex/core/transaction/Transaction.scala b/src/main/scala/scorex/core/transaction/Transaction.scala index ddda38caa4..c170967ad1 100644 --- a/src/main/scala/scorex/core/transaction/Transaction.scala +++ b/src/main/scala/scorex/core/transaction/Transaction.scala @@ -1,6 +1,7 @@ package scorex.core.transaction -import scorex.core.{EphemerealNodeViewModifier, ModifierTypeId} +import org.ergoplatform.modifiers.{NetworkObjectTypeId, TransactionTypeId} +import scorex.core.EphemerealNodeViewModifier import scorex.crypto.hash.Blake2b256 import scorex.util.{ModifierId, bytesToId} @@ -9,14 +10,9 @@ import scorex.util.{ModifierId, bytesToId} * A transaction is an atomic state modifier */ trait Transaction extends EphemerealNodeViewModifier { - override val modifierTypeId: ModifierTypeId = Transaction.ModifierTypeId + override val modifierTypeId: NetworkObjectTypeId.Value = TransactionTypeId.value val messageToSign: Array[Byte] override lazy val id: ModifierId = bytesToId(Blake2b256(messageToSign)) } - - -object Transaction { - val ModifierTypeId: scorex.core.ModifierTypeId = scorex.core.ModifierTypeId @@ 2.toByte -} diff --git a/src/main/scala/scorex/core/validation/ModifierError.scala b/src/main/scala/scorex/core/validation/ModifierError.scala index d8fce64b30..672c93db8f 100644 --- a/src/main/scala/scorex/core/validation/ModifierError.scala +++ b/src/main/scala/scorex/core/validation/ModifierError.scala @@ -1,11 +1,11 @@ package scorex.core.validation -import scorex.core.ModifierTypeId +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.util.ModifierId import scala.util.control.NoStackTrace -case class InvalidModifier(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId) +case class InvalidModifier(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value) /** Base trait for errors that were occurred during NodeView Modifier validation */ @@ -13,7 +13,7 @@ trait ModifierError { def message: String def isFatal: Boolean def modifierId: ModifierId - def modifierTypeId: ModifierTypeId + def modifierTypeId: NetworkObjectTypeId.Value def toThrowable: Throwable def info: String = { @@ -25,7 +25,7 @@ trait ModifierError { /** Permanent modifier error that could not be recovered in future even after any history updates */ @SuppressWarnings(Array("org.wartremover.warts.Null")) -class MalformedModifierError(val message: String, val modifierId: ModifierId, val modifierTypeId: ModifierTypeId, cause: Option[Throwable] = None) +class MalformedModifierError(val message: String, val modifierId: ModifierId, val modifierTypeId: NetworkObjectTypeId.Value, cause: Option[Throwable] = None) extends Exception(message, cause.orNull) with ModifierError { def isFatal: Boolean = true def toThrowable: Throwable = this @@ -35,7 +35,7 @@ class MalformedModifierError(val message: String, val modifierId: ModifierId, va * When an instance is created, the stack trace is not collected which makes this exception lightweight. */ @SuppressWarnings(Array("org.wartremover.warts.Null")) -class RecoverableModifierError(val message: String, val modifierId: ModifierId, val modifierTypeId: ModifierTypeId, cause: Option[Throwable] = None) +class RecoverableModifierError(val message: String, val modifierId: ModifierId, val modifierTypeId: NetworkObjectTypeId.Value, cause: Option[Throwable] = None) extends Exception(message, cause.orNull) with ModifierError with NoStackTrace { def isFatal: Boolean = false def toThrowable: Throwable = this diff --git a/src/main/scala/scorex/core/validation/ModifierValidator.scala b/src/main/scala/scorex/core/validation/ModifierValidator.scala index ed5b4e7ec0..8e23763c69 100644 --- a/src/main/scala/scorex/core/validation/ModifierValidator.scala +++ b/src/main/scala/scorex/core/validation/ModifierValidator.scala @@ -1,7 +1,6 @@ package scorex.core.validation - -import scorex.core.ModifierTypeId +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.core.consensus.ModifierSemanticValidity import scorex.core.utils.ScorexEncoder import scorex.core.validation.ValidationResult._ @@ -29,33 +28,31 @@ object ModifierValidator { } /** report recoverable modifier error that could be fixed by later retries */ - def error(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + def error(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = invalid(new RecoverableModifierError(error, modifierId, modifierTypeId, None)) /** report recoverable modifier error that could be fixed by later retries */ - def error(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId, cause: Throwable): Invalid = + def error(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value, cause: Throwable): Invalid = invalid(new RecoverableModifierError(msg(error, cause), modifierId, modifierTypeId, Option(cause))) /** report recoverable modifier error that could be fixed by later retries */ - def error(description: String, detail: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + def error(description: String, detail: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = error(msg(description, detail), modifierId, modifierTypeId) /** report non-recoverable modifier error that could not be fixed by retries and requires modifier change */ - def fatal(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + def fatal(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = invalid(new MalformedModifierError(error, modifierId, modifierTypeId, None)) /** report non-recoverable modifier error that could not be fixed by retries and requires modifier change */ - def fatal(error: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId, cause: Throwable): Invalid = + def fatal(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value, cause: Throwable): Invalid = invalid(new MalformedModifierError(msg(error, cause), modifierId, modifierTypeId, Option(cause))) /** report non-recoverable modifier error that could not be fixed by retries and requires modifier change */ - def fatal(description: String, detail: String, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + def fatal(description: String, detail: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = fatal(msg(description, detail), modifierId, modifierTypeId) /** unsuccessful validation with a given error; also logs the error as an exception */ - def invalid(error: ModifierError): Invalid = { - Invalid(Seq(error)) - } + def invalid(error: ModifierError): Invalid = Invalid(Seq(error)) /** successful validation without payload */ val success: Valid[Unit] = Valid(()) @@ -103,7 +100,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate the first argument equals the second. This should not be used with `ModifierId` of type `Array[Byte]`. * The `error` callback will be provided with detail on argument values for better reporting */ - def validateEquals[A](id: Short, given: => A, expected: => A, modifierId: ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateEquals[A](id: Short, given: => A, expected: => A, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass((given, expected) match { case _ if !settings.isActive(id) => result case (a: Array[_], b: Array[_]) if a sameElements[Any] b => result @@ -115,7 +112,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate the `id`s are equal. The `error` callback will be provided with detail on argument values */ - def validateEqualIds(id: Short, given: => ModifierId, expected: => ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateEqualIds(id: Short, given: => ModifierId, expected: => ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass { if (!settings.isActive(id) || given == expected) result else settings.getError(id, InvalidModifier(s"Given: ${e.encodeId(given)}, expected ${e.encodeId(expected)}", given, modifierTypeId)) @@ -131,7 +128,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate the `condition` is `Success`. Otherwise the `error` callback will be provided with detail * on a failure exception */ - def validateNoFailure(id: Short, condition: => Try[_], modifierId: ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateNoFailure(id: Short, condition: => Try[_], modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass(if (!settings.isActive(id)) result else condition.fold(e => settings.getError(id, e, modifierId, modifierTypeId), _ => result)) } @@ -143,7 +140,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS * @param modifierTypeId provide for a case when it cannot be resolved from a ModifierError * @return validation state */ - def validateNoFailure(id: Short, condition: => Try[_], modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateNoFailure(id: Short, condition: => Try[_], modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass { if (!settings.isActive(id)) { result @@ -163,7 +160,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate the `block` doesn't throw an Exception. Otherwise the `error` callback will be provided with detail * on the exception */ - def validateNoThrow(id: Short, block: => Any, modifierId: ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateNoThrow(id: Short, block: => Any, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { validateNoFailure(id, Try(block), modifierId, modifierTypeId) } @@ -176,7 +173,8 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate `condition` against payload is `true` or else return the `error` */ - def validateTryFlatten(id: Short, operation: T => Try[T], condition: T => Boolean, modifierId: ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateTryFlatten(id: Short, operation: T => Try[T], condition: T => Boolean, + modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass(result.toTry.flatMap(r => operation(r)) match { case Failure(ex) => settings.getError(id, ex, modifierId, modifierTypeId) case Success(v) if settings.isActive(id) && !condition(v) => settings.getError(id, InvalidModifier(modifierId, modifierId, modifierTypeId)) @@ -200,7 +198,8 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS * If given option is `None` then pass the previous result as success. * Return `error` if option is `Some` amd condition is `false` */ - def validateOrSkipFlatten[A](id: Short, option: => Option[A], condition: A => Boolean, modifierId: ModifierId, modifierTypeId: ModifierTypeId): ValidationState[T] = { + def validateOrSkipFlatten[A](id: Short, option: => Option[A], condition: A => Boolean, + modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { pass(option match { case Some(v) if settings.isActive(id) && !condition(v) => settings.getError(id, InvalidModifier(modifierId, modifierId, modifierTypeId)) case _ => result diff --git a/src/main/scala/scorex/core/validation/ValidationSettings.scala b/src/main/scala/scorex/core/validation/ValidationSettings.scala index 657b4a005e..4ed9bb6cf3 100644 --- a/src/main/scala/scorex/core/validation/ValidationSettings.scala +++ b/src/main/scala/scorex/core/validation/ValidationSettings.scala @@ -1,6 +1,6 @@ package scorex.core.validation -import scorex.core.ModifierTypeId +import org.ergoplatform.modifiers.NetworkObjectTypeId import scorex.core.validation.ValidationResult.Invalid import scorex.util.ModifierId @@ -11,7 +11,7 @@ import scorex.util.ModifierId abstract class ValidationSettings { val isFailFast: Boolean - def getError(id: Short, e: Throwable, modifierId: ModifierId, modifierTypeId: ModifierTypeId): Invalid = + def getError(id: Short, e: Throwable, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = getError(id, InvalidModifier(e.getMessage, modifierId, modifierTypeId)) def getError(id: Short, invalidMod: InvalidModifier): ValidationResult.Invalid diff --git a/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala b/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala index 4a1f08a0f5..16a2389dfa 100644 --- a/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala +++ b/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala @@ -14,7 +14,6 @@ import org.ergoplatform.wallet.protocol.context.{InputContext, TransactionContex import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, Input} import org.scalacheck.Gen import scalan.util.BenchmarkUtil -import scorex.core.transaction.Transaction import scorex.crypto.authds.ADKey import scorex.crypto.hash.{Blake2b256, Digest32} import scorex.db.ByteArrayWrapper @@ -240,7 +239,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { val validity = tx.statefulValidity(from, emptyDataBoxes, emptyStateContext) validity.isSuccess shouldBe false val e = validity.failed.get - e.getMessage should startWith(ValidationRules.errorMessage(ValidationRules.txScriptValidation, "", emptyModifierId, Transaction.ModifierTypeId)) + e.getMessage should startWith(ValidationRules.errorMessage(ValidationRules.txScriptValidation, "", emptyModifierId, ErgoTransaction.modifierTypeId)) } } @@ -317,7 +316,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { } val txMod = tx.copy(inputs = inputsPointers, outputCandidates = out) val validFailure = txMod.statefulValidity(in, emptyDataBoxes, emptyStateContext) - validFailure.failed.get.getMessage should startWith(ValidationRules.errorMessage(txAssetsInOneBox, "", emptyModifierId, Transaction.ModifierTypeId).take(30)) + validFailure.failed.get.getMessage should startWith(ValidationRules.errorMessage(txAssetsInOneBox, "", emptyModifierId, ErgoTransaction.modifierTypeId).take(30)) } property("transaction with too many inputs should be rejected") { @@ -347,7 +346,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { assert(time0 <= Timeout) val cause = validity.failed.get.getMessage - cause should startWith(ValidationRules.errorMessage(bsBlockTransactionsCost, "", emptyModifierId, Transaction.ModifierTypeId).take(30)) + cause should startWith(ValidationRules.errorMessage(bsBlockTransactionsCost, "", emptyModifierId, ErgoTransaction.modifierTypeId).take(30)) //check that spam transaction validation with no cost limit is indeed taking too much time import Parameters._ @@ -535,7 +534,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { val txFailure = tx.statefulValidity(boxes, IndexedSeq.empty, ctx3) txFailure.isSuccess shouldBe false val cause = txFailure.toEither.left.get.getMessage - val expectedMessage = ValidationRules.errorMessage(ValidationRules.txMonotonicHeight, "", emptyModifierId, Transaction.ModifierTypeId) + val expectedMessage = ValidationRules.errorMessage(ValidationRules.txMonotonicHeight, "", emptyModifierId, ErgoTransaction.modifierTypeId) cause should startWith(expectedMessage) } } diff --git a/src/test/scala/org/ergoplatform/nodeView/history/NonVerifyADHistorySpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/NonVerifyADHistorySpecification.scala index 43e3560112..15dd345048 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/NonVerifyADHistorySpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/NonVerifyADHistorySpecification.scala @@ -5,7 +5,6 @@ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.history.popow.NipopowAlgos import org.ergoplatform.modifiers.history.HeaderChain -import org.ergoplatform.modifiers.state.UTXOSnapshotChunk import org.ergoplatform.nodeView.state.StateType import org.ergoplatform.settings.Algos import org.ergoplatform.utils.HistoryTestHelpers @@ -22,15 +21,6 @@ class NonVerifyADHistorySpecification extends HistoryTestHelpers { private lazy val popowHistory = ensureMinimalHeight(genHistory(), 100) - ignore("Should apply UTXOSnapshotChunks") { - forAll(randomUTXOSnapshotChunkGen) { snapshot: UTXOSnapshotChunk => - popowHistory.applicable(snapshot) shouldBe true - val processInfo = popowHistory.append(snapshot).get._2 - processInfo.toApply shouldEqual Some(snapshot) - popowHistory.applicable(snapshot) shouldBe false - } - } - property("Should calculate difficulty correctly") { val epochLength = 3 val useLastEpochs = 3 diff --git a/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala index 4ef3e68b54..30908ceb1a 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala @@ -1,18 +1,17 @@ package org.ergoplatform.nodeView.history -import org.ergoplatform.modifiers.ErgoFullBlock +import org.ergoplatform.modifiers.{ErgoFullBlock, NetworkObjectTypeId} import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.HeaderSerializer -import org.ergoplatform.nodeView.history.storage.modifierprocessors.{FullBlockProcessor, ToDownloadProcessor} +import org.ergoplatform.nodeView.history.storage.modifierprocessors.FullBlockProcessor import org.ergoplatform.nodeView.state.StateType import org.ergoplatform.settings.Algos import org.ergoplatform.utils.HistoryTestHelpers -import scorex.core.ModifierTypeId import scorex.core.consensus.ProgressInfo class VerifyNonADHistorySpecification extends HistoryTestHelpers { - import ToDownloadProcessor._ + import org.ergoplatform.nodeView.history.storage.modifierprocessors.ToDownloadProcessor._ private def genHistory() = generateHistory(verifyTransactions = true, StateType.Utxo, PoPoWBootstrap = false, BlocksToKeep) @@ -118,7 +117,7 @@ class VerifyNonADHistorySpecification extends HistoryTestHelpers { val missedChain = chain.tail.toList val missedBS = missedChain.flatMap { fb => Seq((BlockTransactions.modifierTypeId, fb.blockTransactions.encodedId), (Extension.modifierTypeId, fb.extension.encodedId)) - }.foldLeft(Map.empty[ModifierTypeId, Seq[String]]) { case (newAcc, (mType, mId)) => + }.foldLeft(Map.empty[NetworkObjectTypeId.Value, Seq[String]]) { case (newAcc, (mType, mId)) => newAcc.adjust(mType)(_.fold(Seq(mId))(_ :+ mId)) } diff --git a/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala index 72479d26a3..ec75b9986b 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala @@ -4,7 +4,6 @@ import org.ergoplatform.ErgoBox.TokenId import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction} -import org.ergoplatform.modifiers.state.UTXOSnapshotChunk import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.state.wrapped.WrappedUtxoState import org.ergoplatform.nodeView.state.{BoxHolder, ErgoStateContext, VotingData} @@ -20,7 +19,7 @@ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.wallet.interpreter.TransactionHintsBag import org.ergoplatform.wallet.utils.Generators import org.ergoplatform.{DataInput, ErgoAddress, ErgoAddressEncoder, ErgoBox, ErgoBoxCandidate, Input, P2PKAddress} -import org.scalacheck.{Arbitrary, Gen} +import org.scalacheck.Gen import scorex.crypto.hash.{Blake2b256, Digest32} import scorex.db.ByteArrayWrapper import scorex.util.encode.Base16 @@ -292,11 +291,6 @@ trait ErgoTransactionGenerators extends ErgoGenerators with Generators { } yield BlockTransactions(headerId, Header.InitialVersion, txs.foldLeft(Seq.empty[ErgoTransaction])((acc, tx) => if ((acc :+ tx).map(_.size).sum < (Parameters.MaxBlockSizeDefault - 150)) acc :+ tx else acc)) - lazy val randomUTXOSnapshotChunkGen: Gen[UTXOSnapshotChunk] = for { - index: Short <- Arbitrary.arbitrary[Short] - stateElements: Seq[ErgoBox] <- Gen.listOf(ergoBoxGenNoProp) - } yield UTXOSnapshotChunk(stateElements, index) - lazy val invalidErgoFullBlockGen: Gen[ErgoFullBlock] = for { header <- defaultHeaderGen txs <- invalidBlockTransactionsGen diff --git a/src/test/scala/scorex/core/network/DeliveryTrackerSpec.scala b/src/test/scala/scorex/core/network/DeliveryTrackerSpec.scala index a338df74b2..4c7e5ad532 100644 --- a/src/test/scala/scorex/core/network/DeliveryTrackerSpec.scala +++ b/src/test/scala/scorex/core/network/DeliveryTrackerSpec.scala @@ -3,8 +3,8 @@ package scorex.core.network import akka.actor.{ActorRef, Cancellable} import io.circe._ import io.circe.syntax._ +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.utils.ErgoPropertyTest -import scorex.core.ModifierTypeId import scorex.core.network.ModifiersStatus.Received import scorex.testkit.generators.ObjectGenerators import scorex.util.ModifierId @@ -15,7 +15,7 @@ class DeliveryTrackerSpec extends ErgoPropertyTest with ObjectGenerators { forAll(connectedPeerGen(ActorRef.noSender)) { peer => val tracker = DeliveryTracker.empty(settings) val mid: ModifierId = ModifierId @@ "foo" - val mTypeId: ModifierTypeId = ModifierTypeId @@ (104: Byte) + val mTypeId: NetworkObjectTypeId.Value = NetworkObjectTypeId.fromByte(104) tracker.setRequested(mTypeId, mid, peer) { _ => Cancellable.alreadyCancelled} val infoFields = Seq( diff --git a/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala b/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala index e85cef0572..7c9dddf961 100644 --- a/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala +++ b/src/test/scala/scorex/testkit/generators/ObjectGenerators.scala @@ -3,6 +3,7 @@ package scorex.testkit.generators import java.net.{InetAddress, InetSocketAddress, URL} import akka.actor.ActorRef import akka.util.ByteString +import org.ergoplatform.modifiers.NetworkObjectTypeId import org.ergoplatform.network.ModePeerFeature import org.ergoplatform.nodeView.state.StateType import org.scalacheck.Gen.{const, some} @@ -11,7 +12,7 @@ import scorex.core.app.Version import scorex.core.network.message.{InvData, ModifiersData} import scorex.core.network._ import scorex.core.network.peer.{PeerInfo, RestApiUrlPeerFeature} -import scorex.core.{ModifierTypeId, NodeViewModifier} +import scorex.core.NodeViewModifier import scorex.util.{ModifierId, bytesToId} trait ObjectGenerators { @@ -49,10 +50,10 @@ trait ObjectGenerators { lazy val modifierIdGen: Gen[ModifierId] = Gen.listOfN(NodeViewModifier.ModifierIdSize, Arbitrary.arbitrary[Byte]) .map(id => bytesToId(id.toArray)) - lazy val modifierTypeIdGen: Gen[ModifierTypeId] = Arbitrary.arbitrary[Byte].map(t => ModifierTypeId @@ t) + lazy val modifierTypeIdGen: Gen[NetworkObjectTypeId.Value] = Arbitrary.arbitrary[Byte].map(t => NetworkObjectTypeId.fromByte(t)) lazy val invDataGen: Gen[InvData] = for { - modifierTypeId: ModifierTypeId <- modifierTypeIdGen + modifierTypeId: NetworkObjectTypeId.Value <- modifierTypeIdGen modifierIds: Seq[ModifierId] <- Gen.nonEmptyListOf(modifierIdGen) if modifierIds.nonEmpty } yield InvData(modifierTypeId, modifierIds) @@ -62,7 +63,7 @@ trait ObjectGenerators { } yield id -> mod lazy val modifiersGen: Gen[ModifiersData] = for { - modifierTypeId: ModifierTypeId <- modifierTypeIdGen + modifierTypeId: NetworkObjectTypeId.Value <- modifierTypeIdGen modifiers: Map[ModifierId, Array[Byte]] <- Gen.nonEmptyMap(modifierWithIdGen).suchThat(_.nonEmpty) } yield ModifiersData(modifierTypeId, modifiers) From 0d33ba7973cfb6c37566b907f0623345fa3346a7 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Wed, 25 Jan 2023 15:00:21 +0300 Subject: [PATCH 53/90] 5.0.7 version set --- src/main/resources/api/openapi.yaml | 2 +- src/main/resources/application.conf | 2 +- src/main/resources/mainnet.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index f193638740..0682e970af 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "5.0.4" + version: "5.0.7" title: Ergo Node API description: API docs for Ergo Node. Models are shared between all Ergo products contact: diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 319ad12a33..1aa895d8c0 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -396,7 +396,7 @@ scorex { nodeName = "ergo-node" # Network protocol version to be sent in handshakes - appVersion = 5.0.6 + appVersion = 5.0.7 # Network agent name. May contain information about client code # stack, starting from core code-base up to the end graphical interface. diff --git a/src/main/resources/mainnet.conf b/src/main/resources/mainnet.conf index 4c03289d9b..e84cb490f2 100644 --- a/src/main/resources/mainnet.conf +++ b/src/main/resources/mainnet.conf @@ -80,7 +80,7 @@ scorex { network { magicBytes = [1, 0, 2, 4] bindAddress = "0.0.0.0:9030" - nodeName = "ergo-mainnet-5.0.6" + nodeName = "ergo-mainnet-5.0.7" nodeName = ${?NODENAME} knownPeers = [ "213.239.193.208:9030", From 252d435494e2239059048de2de201e55c77c66ba Mon Sep 17 00:00:00 2001 From: jellymlg Date: Thu, 26 Jan 2023 15:49:46 +0100 Subject: [PATCH 54/90] ModifierTypeId no longer used --- .../history/extra/ExtraIndexSerializer.scala | 24 +++++++++---------- .../nodeView/history/extra/ExtraIndexer.scala | 2 ++ .../history/extra/IndexedErgoAddress.scala | 5 ++-- .../history/extra/IndexedErgoBox.scala | 5 ++-- .../extra/IndexedErgoTransaction.scala | 5 ++-- .../nodeView/history/extra/IndexedToken.scala | 5 ++-- .../nodeView/history/extra/NumericIndex.scala | 7 +++--- 7 files changed, 25 insertions(+), 28 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala index 88592af575..0c22fe4956 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexSerializer.scala @@ -8,22 +8,22 @@ object ExtraIndexSerializer extends ScorexSerializer[ExtraIndex]{ override def serialize(obj: ExtraIndex, w: Writer): Unit = { obj match { case m: IndexedErgoAddress => - w.put(IndexedErgoAddress.modifierTypeId) + w.put(IndexedErgoAddress.extraIndexTypeId) IndexedErgoAddressSerializer.serialize(m, w) case m: IndexedErgoTransaction => - w.put(IndexedErgoTransaction.modifierTypeId) + w.put(IndexedErgoTransaction.extraIndexTypeId) IndexedErgoTransactionSerializer.serialize(m, w) case m: IndexedErgoBox => - w.put(IndexedErgoBox.modifierTypeId) + w.put(IndexedErgoBox.extraIndexTypeId) IndexedErgoBoxSerializer.serialize(m, w) case m: NumericTxIndex => - w.put(NumericTxIndex.modifierTypeId) + w.put(NumericTxIndex.extraIndexTypeId) NumericTxIndexSerializer.serialize(m, w) case m: NumericBoxIndex => - w.put(NumericBoxIndex.modifierTypeId) + w.put(NumericBoxIndex.extraIndexTypeId) NumericBoxIndexSerializer.serialize(m, w) case m: IndexedToken => - w.put(IndexedToken.modifierTypeId) + w.put(IndexedToken.extraIndexTypeId) IndexedTokenSerializer.serialize(m, w) case m => throw new Error(s"Serialization for unknown index: $m") @@ -32,17 +32,17 @@ object ExtraIndexSerializer extends ScorexSerializer[ExtraIndex]{ override def parse(r: Reader): ExtraIndex = { r.getByte() match { - case IndexedErgoAddress.`modifierTypeId` => + case IndexedErgoAddress.`extraIndexTypeId` => IndexedErgoAddressSerializer.parse(r) - case IndexedErgoTransaction.`modifierTypeId` => + case IndexedErgoTransaction.`extraIndexTypeId` => IndexedErgoTransactionSerializer.parse(r) - case IndexedErgoBox.`modifierTypeId` => + case IndexedErgoBox.`extraIndexTypeId` => IndexedErgoBoxSerializer.parse(r) - case NumericTxIndex.`modifierTypeId` => + case NumericTxIndex.`extraIndexTypeId` => NumericTxIndexSerializer.parse(r) - case NumericBoxIndex.`modifierTypeId` => + case NumericBoxIndex.`extraIndexTypeId` => NumericBoxIndexSerializer.parse(r) - case IndexedToken.`modifierTypeId` => + case IndexedToken.`extraIndexTypeId` => IndexedTokenSerializer.parse(r) case m => throw new Error(s"Deserialization for unknown type byte: $m") diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 8004b52869..97f5574e09 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -393,6 +393,8 @@ class ExtraIndexer(cacheSettings: CacheSettings) object ExtraIndexerRef { + type ExtraIndexTypeId = Byte + object ReceivableMessages { /** * Initialize ExtraIndexer and start indexing. diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index c4f421e85d..13bd238ad7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -2,11 +2,10 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{ExtraIndexTypeId, fastIdToBytes} import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getBoxes, getSegmentsForRange, getTxs, segmentTreshold, slice} import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, txSegmentId} import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, ScorexLogging, bytesToId} import scorex.util.serialization.{Reader, Writer} @@ -256,7 +255,7 @@ object IndexedErgoAddressSerializer extends ScorexSerializer[IndexedErgoAddress] object IndexedErgoAddress { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 15.toByte + val extraIndexTypeId: ExtraIndexTypeId = 15.toByte val segmentTreshold: Int = 512 diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala index d1a2d99ee8..f1b3ac1770 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -1,11 +1,10 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.{ErgoAddress, ErgoBox, Pay2SAddress} -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{ExtraIndexTypeId, fastIdToBytes} import org.ergoplatform.nodeView.wallet.WalletBox import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.boxes.{ErgoBoxSerializer, TrackedBox} -import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} @@ -79,5 +78,5 @@ object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { } object IndexedErgoBox { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 5.toByte + val extraIndexTypeId: ExtraIndexTypeId = 5.toByte } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index d1730402a4..090764458f 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -4,9 +4,8 @@ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.DataInput import org.ergoplatform.modifiers.history.BlockTransactions -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{ExtraIndexTypeId, fastIdToBytes} import scorex.core.serialization.ScorexSerializer -import scorex.core.ModifierTypeId import scorex.util.serialization.{Reader, Writer} import scorex.util.{ModifierId, bytesToId} import spire.implicits.cfor @@ -94,5 +93,5 @@ object IndexedErgoTransactionSerializer extends ScorexSerializer[IndexedErgoTran } object IndexedErgoTransaction { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 10.toByte + val extraIndexTypeId: ExtraIndexTypeId = 10.toByte } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index fc58cf714a..0257dec829 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -2,10 +2,9 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.{R4, R5, R6} -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{ExtraIndexTypeId, fastIdToBytes} import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.uniqueId import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} @@ -125,5 +124,5 @@ object IndexedTokenSerializer extends ScorexSerializer[IndexedToken] { } object IndexedToken { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 35.toByte + val extraIndexTypeId: ExtraIndexTypeId = 35.toByte } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala index 4c0852d0a5..706717a322 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala @@ -1,9 +1,8 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.nodeView.history.ErgoHistoryReader -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes +import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{ExtraIndexTypeId, fastIdToBytes} import org.ergoplatform.settings.Algos -import scorex.core.ModifierTypeId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} @@ -32,7 +31,7 @@ object NumericTxIndexSerializer extends ScorexSerializer[NumericTxIndex] { } object NumericTxIndex { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 25.toByte + val extraIndexTypeId: ExtraIndexTypeId = 25.toByte /** * Convert the index number of a transaction to an id for database retreival. @@ -75,7 +74,7 @@ object NumericBoxIndexSerializer extends ScorexSerializer[NumericBoxIndex] { } object NumericBoxIndex { - val modifierTypeId: ModifierTypeId = ModifierTypeId @@ 30.toByte + val extraIndexTypeId: ExtraIndexTypeId = 30.toByte /** * Convert the index number of a box to an id for database retreival. From c09a73b779190ba9c12b67c5076feaa3e915714c Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Thu, 26 Jan 2023 19:17:05 +0300 Subject: [PATCH 55/90] ScalaDoc, ResponseFromLocal removed --- .../ergoplatform/network/ErgoNodeViewSynchronizer.scala | 8 ++++++-- .../nodeView/history/storage/HistoryStorage.scala | 3 +++ .../storage/modifierprocessors/ToDownloadProcessor.scala | 7 +++++-- src/main/scala/scorex/core/NodeViewModifier.scala | 3 +++ .../scala/scorex/core/validation/ValidationSettings.scala | 3 +++ 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 3e7fee5272..6e5d8f9a85 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -1227,8 +1227,6 @@ object ErgoNodeViewSynchronizer { // getLocalSyncInfo messages case object SendLocalSyncInfo - case class ResponseFromLocal(source: ConnectedPeer, modifierTypeId: NetworkObjectTypeId, localObjects: Seq[(ModifierId, Array[Byte])]) - /** * Check delivery of modifier with type `modifierTypeId` and id `modifierId`. * `source` may be defined if we expect modifier from concrete peer or None if @@ -1283,8 +1281,14 @@ object ErgoNodeViewSynchronizer { */ case class FailedOnRecheckTransaction(id : ModifierId, error: Throwable) extends ModificationOutcome + /** + * A signal that block section with id `modifierId` was invalidated due to `error`, but it may be valid in future + */ case class RecoverableFailedModification(typeId: NetworkObjectTypeId.Value, modifierId: ModifierId, error: Throwable) extends ModificationOutcome + /** + * A signal that block section with id `modifierId` was permanently invalidated during stateless checks + */ case class SyntacticallyFailedModification(typeId: NetworkObjectTypeId.Value, modifierId: ModifierId, error: Throwable) extends ModificationOutcome /** diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 1080ae5642..95aae80935 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -55,6 +55,9 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c objectsStore.get(idToBytes(id)).map(_.tail) // removing modifier type byte with .tail } + /** + * @return bytes and type of a network object stored in the database with identifier `id` + */ def modifierTypeAndBytesById(id: ModifierId): Option[(NetworkObjectTypeId.Value, Array[Byte])] = { objectsStore.get(idToBytes(id)).map(bs => (NetworkObjectTypeId.fromByte(bs.head), bs.tail)) // first byte is type id, tail is modifier bytes } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala index 7d5f047141..4b49fcda1c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala @@ -116,11 +116,14 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { } } + /** + * @return block sections needed to be downloaded after header `h` , and defined by the header + */ def requiredModifiersForHeader(h: Header): Seq[(NetworkObjectTypeId.Value, ModifierId)] = { if (!nodeSettings.verifyTransactions) { - Nil + Nil // no block sections to be downloaded in SPV mode } else if (nodeSettings.stateType.requireProofs) { - h.sectionIds + h.sectionIds // download block transactions, extension and UTXO set transformations proofs in "digest" mode } else { h.sectionIdsWithNoProof // do not download UTXO set transformation proofs if UTXO set is stored } diff --git a/src/main/scala/scorex/core/NodeViewModifier.scala b/src/main/scala/scorex/core/NodeViewModifier.scala index 8e3be7e142..c11288a0f6 100644 --- a/src/main/scala/scorex/core/NodeViewModifier.scala +++ b/src/main/scala/scorex/core/NodeViewModifier.scala @@ -17,6 +17,9 @@ sealed trait NodeViewModifier extends BytesSerializable with ScorexEncoding {sel case _ => false } + /** + * @return readable representation of `id`, as `id` is a hex-encoded string now, just identity functions is used + */ def encodedId: String = id } diff --git a/src/main/scala/scorex/core/validation/ValidationSettings.scala b/src/main/scala/scorex/core/validation/ValidationSettings.scala index 4ed9bb6cf3..cabd24da9d 100644 --- a/src/main/scala/scorex/core/validation/ValidationSettings.scala +++ b/src/main/scala/scorex/core/validation/ValidationSettings.scala @@ -11,6 +11,9 @@ import scorex.util.ModifierId abstract class ValidationSettings { val isFailFast: Boolean + /** + * @return validation error of type `id` for block section `modifierId` of type `modifierTypeId`, error details in `e` + */ def getError(id: Short, e: Throwable, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value): Invalid = getError(id, InvalidModifier(e.getMessage, modifierId, modifierTypeId)) From efac9b4a7a450c0456aca53f13f1f2d3f3a5cced Mon Sep 17 00:00:00 2001 From: jellymlg Date: Thu, 26 Jan 2023 19:26:50 +0100 Subject: [PATCH 56/90] reworked address encoding --- .../ergoplatform/bench/HistoryExtractor.scala | 2 +- src/main/scala/org/ergoplatform/ErgoApp.scala | 9 +-- .../org/ergoplatform/http/api/ApiCodecs.scala | 9 ++- .../http/api/BlockchainApiRoute.scala | 10 +-- .../http/api/EmissionApiRoute.scala | 2 +- .../nodeView/history/ErgoHistory.scala | 12 ++-- .../nodeView/history/extra/BalanceInfo.scala | 9 ++- .../nodeView/history/extra/ExtraIndexer.scala | 63 ++++++++----------- .../history/extra/IndexedErgoAddress.scala | 6 +- .../history/extra/IndexedErgoBox.scala | 15 +---- .../extra/IndexedErgoTransaction.scala | 2 +- .../nodeView/history/extra/IndexedToken.scala | 2 +- .../nodeView/history/extra/NumericIndex.scala | 2 +- .../tools/DifficultyEstimation.scala | 4 +- ...rgoNodeViewSynchronizerSpecification.scala | 8 +-- .../extra/ExtraIndexerSpecification.scala | 6 +- .../ergoplatform/tools/ChainGenerator.scala | 2 +- .../utils/HistoryTestHelpers.scala | 2 +- .../scala/org/ergoplatform/utils/Stubs.scala | 2 +- 19 files changed, 72 insertions(+), 95 deletions(-) diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala index 027bafa170..c067709ae2 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala @@ -22,7 +22,7 @@ object HistoryExtractor extends ScorexLogging { val timeProvider = new NetworkTimeProvider(settings.ntp) val os = new FileOutputStream(outputFile) - val h = ErgoHistory.readOrGenerate(ergoSettings, timeProvider) + val h = ErgoHistory.readOrGenerate(ergoSettings, timeProvider)(null) val wholeChain = h.chainToHeader(None, h.bestHeaderOpt.get) var counter = 0 diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index 58781ec162..7249c07922 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -11,7 +11,7 @@ import org.ergoplatform.mining.ErgoMiner import org.ergoplatform.mining.ErgoMiner.StartMining import org.ergoplatform.network.{ErgoNodeViewSynchronizer, ErgoSyncTracker} import org.ergoplatform.nodeView.history.ErgoSyncInfoMessageSpec -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef +import org.ergoplatform.nodeView.history.extra.ExtraIndexer import org.ergoplatform.nodeView.{ErgoNodeViewRef, ErgoReadersHolderRef} import org.ergoplatform.settings.{Args, ErgoSettings, NetworkType} import scorex.core.api.http._ @@ -99,11 +99,8 @@ class ErgoApp(args: Args) extends ScorexLogging { None } - // Create an instance of ExtraIndexer actor if "extraIndex = true" in config - if(ergoSettings.nodeSettings.extraIndex) - ExtraIndexerRef(ergoSettings.chainSettings, ergoSettings.cacheSettings) - - ExtraIndexerRef.setAddressEncoder(ergoSettings.addressEncoder) // initialize an accessible address encoder regardless of extra indexing being enabled + // Create an instance of ExtraIndexer actor (will start if "extraIndex = true" in config) + ExtraIndexer(ergoSettings.chainSettings, ergoSettings.cacheSettings) private val syncTracker = ErgoSyncTracker(scorexSettings.network, timeProvider) diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index ee1278bda2..cef83cbb00 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -3,7 +3,7 @@ package org.ergoplatform.http.api import java.math.BigInteger import io.circe._ import org.bouncycastle.util.BigIntegers -import org.ergoplatform.{ErgoBox, ErgoLikeContext, ErgoLikeTransaction, JsonCodecs, UnsignedErgoLikeTransaction} +import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, ErgoLikeContext, ErgoLikeTransaction, JsonCodecs, UnsignedErgoLikeTransaction} import org.ergoplatform.http.api.ApiEncoderOption.Detalization import org.ergoplatform.ErgoBox.RegisterId import org.ergoplatform.mining.{groupElemFromBytes, groupElemToBytes} @@ -28,7 +28,8 @@ import sigmastate.interpreter._ import sigmastate.interpreter.CryptoConstants.EcPointType import io.circe.syntax._ import org.ergoplatform.http.api.requests.{CryptoResult, ExecuteRequest, HintExtractionRequest} -import org.ergoplatform.nodeView.history.extra.{BalanceInfo, ExtraIndexerRef, IndexedErgoBox, IndexedErgoTransaction, IndexedToken} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.getAddress +import org.ergoplatform.nodeView.history.extra.{BalanceInfo, IndexedErgoBox, IndexedErgoTransaction, IndexedToken} import org.ergoplatform.wallet.interface4j.SecretString import scorex.crypto.authds.{LeafData, Side} import scorex.crypto.authds.merkle.MerkleProof @@ -52,6 +53,8 @@ trait ApiCodecs extends JsonCodecs { implicit val sideEncoder: Encoder[Side] = _.toByte.asJson + implicit val ergoAddressEncoder: ErgoAddressEncoder = null + protected implicit def merkleProofEncoder[D <: Digest]: Encoder[MerkleProof[D]] = { proof => Json.obj( "leafData" -> proof.leafData.asJson, @@ -474,7 +477,7 @@ trait ApiCodecs extends JsonCodecs { iEb.box.asJson.deepMerge(Json.obj( "globalIndex" -> iEb.globalIndex.asJson, "inclusionHeight" -> iEb.inclusionHeight.asJson, - "address" -> ExtraIndexerRef.getAddressEncoder.toString(iEb.getAddress).asJson, + "address" -> ergoAddressEncoder.toString(getAddress(iEb.box.ergoTree)).asJson, "spentTransactionId" -> iEb.spendingTxIdOpt.asJson )) } diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 8c29cc051f..2828400c0a 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -8,7 +8,7 @@ import io.circe.syntax.EncoderOps import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder} import org.ergoplatform.nodeView.ErgoReadersHolder.{GetDataFromHistory, GetReaders, Readers} import org.ergoplatform.nodeView.history.ErgoHistoryReader -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{GlobalBoxIndexKey, GlobalTxIndexKey} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{GlobalBoxIndexKey, GlobalTxIndexKey} import org.ergoplatform.nodeView.history.extra._ import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader import org.ergoplatform.settings.ErgoSettings @@ -32,12 +32,12 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private val MaxItems = 16384 - implicit val ae: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder + override implicit val ergoAddressEncoder: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder private val ergoAddress: Directive1[ErgoAddress] = entity(as[String]).flatMap(handleErgoAddress) private def handleErgoAddress(value: String): Directive1[ErgoAddress] = { - ae.fromString(value) match { + ergoAddressEncoder.fromString(value) match { case Success(addr) => provide(addr) case _ => reject(ValidationRejection("Wrong address format")) } @@ -86,7 +86,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private def getIndexedHeightF: Future[Json] = getHistory.map { history => Json.obj( - "indexedHeight" -> ExtraIndexerRef.getIndex(ExtraIndexerRef.IndexedHeightKey)(history).getInt().asJson, + "indexedHeight" -> ExtraIndexer.getIndex(ExtraIndexer.IndexedHeightKey)(history).getInt().asJson, "fullHeight" -> history.fullBlockHeight.asJson ) } @@ -271,7 +271,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting val bal: BalanceInfo = new BalanceInfo mempool.getAll.map(_.transaction).foreach(tx => { tx.outputs.foreach(box => { - if(IndexedErgoBoxSerializer.getAddress(box.ergoTree).equals(address)) bal.add(box) + if(address.equals(ExtraIndexer.getAddress(box.ergoTree))) bal.add(box) }) }) bal diff --git a/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala index ea27de357a..0922a83a8f 100644 --- a/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala @@ -22,7 +22,7 @@ case class EmissionApiRoute(ergoSettings: ErgoSettings) private val reemissionSettings = chainSettings.reemission - private implicit val ergoAddressEncoder: ErgoAddressEncoder = new ErgoAddressEncoder(chainSettings.addressPrefix) + override implicit val ergoAddressEncoder: ErgoAddressEncoder = new ErgoAddressEncoder(chainSettings.addressPrefix) override def route: Route = pathPrefix("emission") { emissionAt ~ scripts diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala index 8d3c33505e..46d29c9e71 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala @@ -1,13 +1,14 @@ package org.ergoplatform.nodeView.history -import java.io.File +import akka.actor.ActorContext +import java.io.File import org.ergoplatform.ErgoLikeContext import org.ergoplatform.mining.AutolykosPowScheme import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.header.{Header, PreGenesisHeader} -import org.ergoplatform.modifiers.{NonHeaderBlockSection, ErgoFullBlock, BlockSection} -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRefHolder +import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock, NonHeaderBlockSection} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.ReceivableMessages.StartExtraIndexer import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.nodeView.history.storage.modifierprocessors._ import org.ergoplatform.settings._ @@ -270,7 +271,7 @@ object ErgoHistory extends ScorexLogging { } } - def readOrGenerate(ergoSettings: ErgoSettings, ntp: NetworkTimeProvider): ErgoHistory = { + def readOrGenerate(ergoSettings: ErgoSettings, ntp: NetworkTimeProvider)(implicit context: ActorContext): ErgoHistory = { val db = HistoryStorage(ergoSettings) val nodeSettings = ergoSettings.nodeSettings @@ -311,7 +312,8 @@ object ErgoHistory extends ScorexLogging { repairIfNeeded(history) log.info("History database read") - ExtraIndexerRefHolder.start(history) //start extra indexer, if enabled + if(ergoSettings.nodeSettings.extraIndex) // start extra indexer, if enabled + context.system.eventStream.publish(StartExtraIndexer(history)) history } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala index abd9c7ff63..10c0cfd364 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -1,9 +1,8 @@ package org.ergoplatform.nodeView.history.extra -import org.ergoplatform.ErgoBox +import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} import org.ergoplatform.nodeView.history.ErgoHistoryReader -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.fastIdToBytes -import org.ergoplatform.nodeView.history.extra.IndexedErgoBoxSerializer.getAddress +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.fastIdToBytes import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.uniqueId import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, ScorexLogging, bytesToId} @@ -44,7 +43,7 @@ class BalanceInfo(var nanoErgs: Long = 0L, } } - def subtract(box: ErgoBox): Unit = { + def subtract(box: ErgoBox)(implicit ae: ErgoAddressEncoder): Unit = { nanoErgs = math.max(nanoErgs - box.value, 0) cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => val id: ModifierId = bytesToId(box.additionalTokens(i)._1) @@ -55,7 +54,7 @@ class BalanceInfo(var nanoErgs: Long = 0L, tokens.remove(n) else tokens(n) = (id, newVal) - case None => log.warn(s"Failed to subtract token $id from address ${getAddress(box.ergoTree)}") + case None => log.warn(s"Failed to subtract token $id from address ${ae.fromProposition(box.ergoTree).map(ae.toString).getOrElse(box.ergoTree.bytesHex)}") } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 97f5574e09..f595c30651 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -2,20 +2,21 @@ package org.ergoplatform.nodeView.history.extra import akka.actor.{Actor, ActorRef, ActorSystem, Props} import org.ergoplatform.ErgoBox.{BoxId, TokenId} -import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, ErgoBox, Pay2SAddress} import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.{FullBlockApplied, Rollback} -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{GlobalBoxIndexKey, GlobalTxIndexKey, IndexedHeightKey, getIndex} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{GlobalBoxIndexKey, GlobalTxIndexKey, IndexedHeightKey, getIndex} import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.ReceivableMessages.Start +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.ReceivableMessages.StartExtraIndexer import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.segmentTreshold import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.hashErgoTree import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.tokenRegistersSet import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.settings.{Algos, CacheSettings, ChainSettings} import scorex.util.{ModifierId, ScorexLogging, bytesToId} +import sigmastate.Values.ErgoTree import java.nio.ByteBuffer import scala.collection.mutable.{ArrayBuffer, ListBuffer} @@ -44,6 +45,9 @@ trait ExtraIndexerBase extends ScorexLogging { // Max buffer size (determined by config) protected val saveLimit: Int + // Address encoder instance + protected implicit val addressEncoder: ErgoAddressEncoder + // Flag to signal when indexer has reached current block height protected var caughtUp: Boolean = false @@ -358,14 +362,18 @@ trait ExtraIndexerBase extends ScorexLogging { * Actor that constructs an index of database elements. * @param cacheSettings - cacheSettings to use for saveLimit size */ -class ExtraIndexer(cacheSettings: CacheSettings) +class ExtraIndexer(cacheSettings: CacheSettings, + ae: ErgoAddressEncoder) extends Actor with ExtraIndexerBase { override val saveLimit: Int = cacheSettings.history.extraCacheSize * 10 + override implicit val addressEncoder: ErgoAddressEncoder = ae + override def preStart(): Unit = { context.system.eventStream.subscribe(self, classOf[FullBlockApplied]) context.system.eventStream.subscribe(self, classOf[Rollback]) + context.system.eventStream.subscribe(self, classOf[StartExtraIndexer]) } override def postStop(): Unit = @@ -384,14 +392,14 @@ class ExtraIndexer(cacheSettings: CacheSettings) run() // restart indexer } - case Start(history: ErgoHistory) => + case StartExtraIndexer(history: ErgoHistory) => _history = history run() } } -object ExtraIndexerRef { +object ExtraIndexer { type ExtraIndexTypeId = Byte @@ -400,14 +408,17 @@ object ExtraIndexerRef { * Initialize ExtraIndexer and start indexing. * @param history - handle to database */ - case class Start(history: ErgoHistory) + case class StartExtraIndexer(history: ErgoHistory) } - private var _ae: ErgoAddressEncoder = _ - - def getAddressEncoder: ErgoAddressEncoder = _ae - - def setAddressEncoder(encoder: ErgoAddressEncoder): Unit = if(_ae == null) _ae = encoder + /** + * @return address constructed from the ErgoTree of this box + */ + def getAddress(tree: ErgoTree)(implicit ae: ErgoAddressEncoder): ErgoAddress = + tree.root match { + case Right(_) => ae.fromProposition(tree).get // default most of the time + case Left(_) => new Pay2SAddress(tree, tree.bytes) // needed for burn address 4MQyMKvMbnCJG3aJ + } private val hexIndex: Array[Byte] = { val index = Array.fill[Byte](128)(0xff.toByte) @@ -438,34 +449,10 @@ object ExtraIndexerRef { def getIndex(key: Array[Byte])(history: ErgoHistoryReader): ByteBuffer = ByteBuffer.wrap(history.modifierBytesById(bytesToId(key)).getOrElse(Array.fill[Byte](8){0})) - def apply(chainSettings: ChainSettings, cacheSettings: CacheSettings)(implicit system: ActorSystem): ActorRef = { - val actor = system.actorOf(Props.create(classOf[ExtraIndexer], cacheSettings)) - _ae = chainSettings.addressEncoder - ExtraIndexerRefHolder.init(actor) - actor - } + def apply(chainSettings: ChainSettings, cacheSettings: CacheSettings)(implicit system: ActorSystem): ActorRef = + system.actorOf(Props.create(classOf[ExtraIndexer], cacheSettings, chainSettings.addressEncoder)) } -object ExtraIndexerRefHolder { - - private var _actor: Option[ActorRef] = None - private var running: Boolean = false - - /** - * Start stored ExtraIndexer actor with history handle. - * @param history - initialized history handle - */ - private[history] def start(history: ErgoHistory): Unit = if(_actor.isDefined && !running) { - _actor.get ! Start(history) - running = true - } - - /** - * Store a Ref to an ExtraIndexer actor. - * @param actor - Ref to ExtraIndexer - */ - private[extra] def init(actor: ActorRef): Unit = _actor = Some(actor) -} /** * Base trait for all additional indexes made by ExtraIndexer */ diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala index 13bd238ad7..4a9f0f1103 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoAddress.scala @@ -1,8 +1,8 @@ package org.ergoplatform.nodeView.history.extra -import org.ergoplatform.ErgoBox +import org.ergoplatform.{ErgoAddressEncoder, ErgoBox} import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{ExtraIndexTypeId, fastIdToBytes} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes} import org.ergoplatform.nodeView.history.extra.IndexedErgoAddress.{getBoxes, getSegmentsForRange, getTxs, segmentTreshold, slice} import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, txSegmentId} import org.ergoplatform.settings.Algos @@ -126,7 +126,7 @@ case class IndexedErgoAddress(treeHash: ModifierId, * @param box - box to spend * @return this address */ - private[extra] def spendBox(box: ErgoBox): IndexedErgoAddress = { + private[extra] def spendBox(box: ErgoBox)(implicit ae: ErgoAddressEncoder): IndexedErgoAddress = { balanceInfo.get.subtract(box) this } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala index f1b3ac1770..9d70620a0f 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -1,14 +1,13 @@ package org.ergoplatform.nodeView.history.extra -import org.ergoplatform.{ErgoAddress, ErgoBox, Pay2SAddress} -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{ExtraIndexTypeId, fastIdToBytes} +import org.ergoplatform.ErgoBox +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes} import org.ergoplatform.nodeView.wallet.WalletBox import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.boxes.{ErgoBoxSerializer, TrackedBox} import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} -import sigmastate.Values.ErgoTree /** * Index of a box. @@ -34,10 +33,6 @@ class IndexedErgoBox(val inclusionHeight: Int, override def serializedId: Array[Byte] = box.id - /** - * @return address constructed from the ErgoTree of this box - */ - def getAddress: ErgoAddress = IndexedErgoBoxSerializer.getAddress(box.ergoTree) /** * Fill in spending parameters. @@ -53,12 +48,6 @@ class IndexedErgoBox(val inclusionHeight: Int, } object IndexedErgoBoxSerializer extends ScorexSerializer[IndexedErgoBox] { - def getAddress(tree: ErgoTree): ErgoAddress = - tree.root match { - case Right(_) => ExtraIndexerRef.getAddressEncoder.fromProposition(tree).get // default most of the time - case Left(_) => new Pay2SAddress(tree, tree.bytes)(ExtraIndexerRef.getAddressEncoder) // needed for burn address 4MQyMKvMbnCJG3aJ - } - override def serialize(iEb: IndexedErgoBox, w: Writer): Unit = { w.putInt(iEb.inclusionHeight) w.putOption[ModifierId](iEb.spendingTxIdOpt)((ww, id) => ww.putBytes(fastIdToBytes(id))) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index 090764458f..aba5ff6f84 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -4,7 +4,7 @@ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.DataInput import org.ergoplatform.modifiers.history.BlockTransactions -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{ExtraIndexTypeId, fastIdToBytes} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes} import scorex.core.serialization.ScorexSerializer import scorex.util.serialization.{Reader, Writer} import scorex.util.{ModifierId, bytesToId} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 0257dec829..c0bc43d0a6 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -2,7 +2,7 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.{R4, R5, R6} -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{ExtraIndexTypeId, fastIdToBytes} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes} import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.uniqueId import org.ergoplatform.settings.Algos import scorex.core.serialization.ScorexSerializer diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala index 706717a322..ab519316bf 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/NumericIndex.scala @@ -1,7 +1,7 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.nodeView.history.ErgoHistoryReader -import org.ergoplatform.nodeView.history.extra.ExtraIndexerRef.{ExtraIndexTypeId, fastIdToBytes} +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, fastIdToBytes} import org.ergoplatform.settings.Algos import scorex.core.serialization.ScorexSerializer import scorex.util.{ModifierId, bytesToId} diff --git a/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala b/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala index 288150c80e..978e42a37f 100644 --- a/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala +++ b/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala @@ -20,7 +20,7 @@ object v2testing extends App { val ldc = new DifficultyAdjustment(ergoSettings.chainSettings) - val eh = ErgoHistory.readOrGenerate(ergoSettings, ntp) + val eh = ErgoHistory.readOrGenerate(ergoSettings, ntp)(null) val epochLength = ergoSettings.chainSettings.epochLength @@ -103,7 +103,7 @@ object AltDiff extends App { println(currentSettings.chainSettings.epochLength) println(altSettings.chainSettings.epochLength) - val eh = ErgoHistory.readOrGenerate(altSettings, ntp) + val eh = ErgoHistory.readOrGenerate(altSettings, ntp)(null) println("best: " + eh.bestHeaderOpt.map(_.height)) diff --git a/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala b/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala index 5e8349e802..6cf8809cad 100644 --- a/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala +++ b/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala @@ -254,7 +254,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc implicit val patienceConfig: PatienceConfig = PatienceConfig(5.second, 100.millis) // we generate and apply existing base chain - val hhistory = ErgoHistory.readOrGenerate(settings, timeProvider) + val hhistory = ErgoHistory.readOrGenerate(settings, timeProvider)(null) val baseChain = genHeaderChain(_.size > 4, None, hhistory.difficultyCalculator, None, false) baseChain.headers.foreach(hhistory.append) val bestHeaderOpt = hhistory.bestHeaderOpt @@ -290,7 +290,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc } eventually { // test whether applied header was actually persisted to history - val hist = ErgoHistory.readOrGenerate(settings, timeProvider) + val hist = ErgoHistory.readOrGenerate(settings, timeProvider)(null) hist.bestHeaderIdOpt.get shouldBe appliedHeader.id } } @@ -313,7 +313,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc // we generate fork of two headers, starting from the parent of the best header // so the depth of the rollback is 1, and the fork bypasses the best chain by 1 header - val hhistory = ErgoHistory.readOrGenerate(settings, timeProvider) + val hhistory = ErgoHistory.readOrGenerate(settings, timeProvider)(null) val newHeaders = genHeaderChain(2, hhistory, diffBitsOpt = None, useRealTs = false).headers val newHistory = newHeaders.foldLeft(hhistory) { case (hist, header) => hist.append(header).get._1 } val parentOpt = newHistory.lastHeaders(2).headOption @@ -354,7 +354,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc deliveryTracker.reset() - val hist = ErgoHistory.readOrGenerate(settings, timeProvider) + val hist = ErgoHistory.readOrGenerate(settings, timeProvider)(null) // generate smaller fork that is going to be reverted after applying a bigger fork val smallFork = genChain(4, hist) diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala index d717629aa7..9a23dd8986 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala @@ -46,11 +46,11 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, initSettings.chainSettings, nodeSettings, settings.scorexSettings, settings.walletSettings, settings.cacheSettings) - _history = ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + _history = ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider)(null) ChainGenerator.generate(HEIGHT, dir)(_history) - ExtraIndexerRef.setAddressEncoder(initSettings.chainSettings.addressEncoder) + ExtraIndexer.setAddressEncoder(initSettings.chainSettings.addressEncoder) run() @@ -72,7 +72,7 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w }) } tx.outputs.foreach(output => { boxesIndexed += 1 - val address: ErgoAddress = IndexedErgoBoxSerializer.getAddress(output.ergoTree) + val address: ErgoAddress = initSettings.chainSettings.addressEncoder.fromProposition(output.ergoTree).get addresses.put(address, addresses.getOrElse[Long](address, 0) + output.value) }) }) diff --git a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala index 9d168632ec..fc93c59249 100644 --- a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala +++ b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala @@ -76,7 +76,7 @@ object ChainGenerator extends App with ErgoTestHelpers { val votingEpochLength = votingSettings.votingLength val protocolVersion = fullHistorySettings.chainSettings.protocolVersion - val history = ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + val history = ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider)(null) HistoryTestHelpers.allowToApplyOldBlocks(history) val (state, _) = ErgoState.generateGenesisUtxoState(stateDir, StateConstants(fullHistorySettings)) log.info(s"Going to generate a chain at ${dir.getAbsoluteFile} starting from ${history.bestFullBlockOpt}") diff --git a/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala b/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala index bfc67cc374..48d816c18d 100644 --- a/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala +++ b/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala @@ -65,7 +65,7 @@ trait HistoryTestHelpers extends ErgoPropertyTest { val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, chainSettings, nodeSettings, scorexSettings, walletSettings, settings.cacheSettings) - ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider)(null) } } diff --git a/src/test/scala/org/ergoplatform/utils/Stubs.scala b/src/test/scala/org/ergoplatform/utils/Stubs.scala index 5851909fb7..4f3a7d27bf 100644 --- a/src/test/scala/org/ergoplatform/utils/Stubs.scala +++ b/src/test/scala/org/ergoplatform/utils/Stubs.scala @@ -383,7 +383,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, chainSettings, nodeSettings, scorexSettings, walletSettings, settings.cacheSettings) - ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider)(null) } def syntacticallyValidModifier(history: HT): Header = { From 7e723b982f46e7aec8f071d3640dc50568824236 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Thu, 26 Jan 2023 19:40:25 +0100 Subject: [PATCH 57/90] fixed tests --- src/it/scala/org/ergoplatform/it/WalletSpec.scala | 4 ++-- .../http/routes/ScriptApiRouteSpec.scala | 14 +++++++------- .../http/routes/WalletApiRouteSpec.scala | 6 +++--- .../history/extra/ExtraIndexerSpecification.scala | 9 ++++----- .../generators/ErgoTransactionGenerators.scala | 2 +- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/it/scala/org/ergoplatform/it/WalletSpec.scala b/src/it/scala/org/ergoplatform/it/WalletSpec.scala index 9c511d014c..a3c6b4012a 100644 --- a/src/it/scala/org/ergoplatform/it/WalletSpec.scala +++ b/src/it/scala/org/ergoplatform/it/WalletSpec.scala @@ -79,8 +79,8 @@ class WalletSpec extends AsyncWordSpec with IntegrationSuite with WalletTestOps val encodedBox = Base16.encode(ErgoBoxSerializer.toBytes(input)) - val paymentRequest = PaymentRequest(P2PKAddress(pk), 50000000, Seq.empty, Map.empty) - val requestsHolder = RequestsHolder(Seq(paymentRequest), feeOpt = Some(100000L), Seq(encodedBox), dataInputsRaw = Seq.empty, minerRewardDelay = 720) + val paymentRequest = PaymentRequest(P2PKAddress(pk)(addressEncoder), 50000000, Seq.empty, Map.empty) + val requestsHolder = RequestsHolder(Seq(paymentRequest), feeOpt = Some(100000L), Seq(encodedBox), dataInputsRaw = Seq.empty, minerRewardDelay = 720)(addressEncoder) node.waitForStartup.flatMap { node: Node => for { diff --git a/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala index a865e9708f..f261e1769a 100644 --- a/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/ScriptApiRouteSpec.scala @@ -67,7 +67,7 @@ class ScriptApiRouteSpec extends AnyFlatSpec val assertion = (json: Json) => { status shouldBe StatusCodes.OK val addressStr = json.hcursor.downField("address").as[String].right.get - ergoAddressEncoder.fromString(addressStr).get.addressTypePrefix shouldEqual Pay2SAddress.addressTypePrefix + addressEncoder.fromString(addressStr).get.addressTypePrefix shouldEqual Pay2SAddress.addressTypePrefix } Post(prefix + suffix, Json.obj("source" -> scriptSource.asJson)) ~> route ~> check(assertion(responseAs[Json])) Post(prefix + suffix, Json.obj("source" -> scriptSourceSigProp.asJson)) ~> route ~> @@ -97,9 +97,9 @@ class ScriptApiRouteSpec extends AnyFlatSpec val tree = ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(Base16.decode(treeStr).get) - val addr = ergoAddressEncoder.fromProposition(tree).get + val addr = addressEncoder.fromProposition(tree).get - ergoAddressEncoder.toString(addr) shouldBe address + addressEncoder.toString(addr) shouldBe address } val p2pk = "3WvsT2Gm4EpsM9Pg18PdY6XyhNNMqXDsvJTbbf6ihLvAmSb7u5RN" @@ -111,7 +111,7 @@ class ScriptApiRouteSpec extends AnyFlatSpec val script = TrueLeaf val tree = ErgoTree.fromProposition(script) - val p2s = ergoAddressEncoder.toString(ergoAddressEncoder.fromProposition(tree).get) + val p2s = addressEncoder.toString(addressEncoder.fromProposition(tree).get) p2s shouldBe "Ms7smJwLGbUAjuWQ" Get(s"$prefix/$suffix/$p2s") ~> route ~> check(assertion(responseAs[Json], p2s)) } @@ -130,9 +130,9 @@ class ScriptApiRouteSpec extends AnyFlatSpec val tree = ErgoTreeSerializer.DefaultSerializer.deserializeErgoTree(bs) - val addr = ergoAddressEncoder.fromProposition(tree).get + val addr = addressEncoder.fromProposition(tree).get - ergoAddressEncoder.toString(addr) shouldBe address + addressEncoder.toString(addr) shouldBe address } val p2pk = "3WvsT2Gm4EpsM9Pg18PdY6XyhNNMqXDsvJTbbf6ihLvAmSb7u5RN" @@ -144,7 +144,7 @@ class ScriptApiRouteSpec extends AnyFlatSpec val script = TrueLeaf val tree = ErgoTree.fromProposition(script) - val p2s = ergoAddressEncoder.toString(ergoAddressEncoder.fromProposition(tree).get) + val p2s = addressEncoder.toString(addressEncoder.fromProposition(tree).get) p2s shouldBe "Ms7smJwLGbUAjuWQ" Get(s"$prefix/$suffix/$p2s") ~> route ~> check(assertion(responseAs[Json], p2s)) } diff --git a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala index df998daf35..3aa73d9dc1 100644 --- a/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/WalletApiRouteSpec.scala @@ -46,15 +46,15 @@ class WalletApiRouteSpec extends AnyFlatSpec implicit val requestsHolderEncoder: RequestsHolderEncoder = new RequestsHolderEncoder(ergoSettings) implicit val addressJsonDecoder: Decoder[ErgoAddress] = ErgoAddressJsonEncoder(settings).decoder - val paymentRequest = PaymentRequest(Pay2SAddress(Constants.FalseLeaf), 100L, Seq.empty, Map.empty) - val assetIssueRequest = AssetIssueRequest(Pay2SAddress(Constants.FalseLeaf), None, 100L, "TEST", "Test", 8) + val paymentRequest = PaymentRequest(Pay2SAddress(Constants.FalseLeaf)(addressEncoder), 100L, Seq.empty, Map.empty) + val assetIssueRequest = AssetIssueRequest(Pay2SAddress(Constants.FalseLeaf)(addressEncoder), None, 100L, "TEST", "Test", 8) val requestsHolder = RequestsHolder( (0 to 10).flatMap(_ => Seq(paymentRequest, assetIssueRequest)), Some(10000L), Seq.empty, Seq.empty, minerRewardDelay = 720 - ) + )(addressEncoder) it should "generate arbitrary transaction" in { diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala index 9a23dd8986..410859d1df 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala @@ -1,6 +1,6 @@ package org.ergoplatform.nodeView.history.extra -import org.ergoplatform.{ErgoAddress, ErgoBox, ErgoBoxCandidate, ErgoScriptPredef, P2PKAddress, UnsignedInput} +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, ErgoBox, ErgoBoxCandidate, ErgoScriptPredef, P2PKAddress, UnsignedInput} import org.ergoplatform.ErgoLikeContext.Height import org.ergoplatform.mining.difficulty.RequiredDifficulty import org.ergoplatform.mining.{AutolykosPowScheme, CandidateBlock, CandidateGenerator} @@ -28,6 +28,7 @@ import scala.util.Try class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase with HistoryTestHelpers { override protected val saveLimit: Int = 1 // save every block + override protected implicit val addressEncoder: ErgoAddressEncoder = initSettings.chainSettings.addressEncoder val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(StateType.Utxo, verifyTransactions = true, -1, poPoWBootstrap = false, ChainGenerator.minimalSuffix, mining = false, ChainGenerator.txCostLimit, ChainGenerator.txSizeLimit, useExternalMiner = false, @@ -50,8 +51,6 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w ChainGenerator.generate(HEIGHT, dir)(_history) - ExtraIndexer.setAddressEncoder(initSettings.chainSettings.addressEncoder) - run() val txIndexBefore = globalTxIndex @@ -67,12 +66,12 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w if(i != 1) { tx.inputs.foreach(input => { val iEb: IndexedErgoBox = _history.getReader.typedExtraIndexById[IndexedErgoBox](bytesToId(input.boxId)).get - val address: ErgoAddress = iEb.getAddress + val address: ErgoAddress = ExtraIndexer.getAddress(iEb.box.ergoTree) addresses.put(address, addresses(address) - iEb.box.value) }) } tx.outputs.foreach(output => { boxesIndexed += 1 - val address: ErgoAddress = initSettings.chainSettings.addressEncoder.fromProposition(output.ergoTree).get + val address: ErgoAddress = addressEncoder.fromProposition(output.ergoTree).get addresses.put(address, addresses.getOrElse[Long](address, 0) + output.value) }) }) diff --git a/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala index 72479d26a3..b42d9b1806 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala @@ -36,7 +36,7 @@ import scala.util.Random trait ErgoTransactionGenerators extends ErgoGenerators with Generators { - protected implicit val ergoAddressEncoder: ErgoAddressEncoder = + protected implicit val addressEncoder: ErgoAddressEncoder = ErgoAddressEncoder(settings.chainSettings.addressPrefix) val creationHeightGen: Gen[Int] = Gen.choose(0, Int.MaxValue / 2) From 23e88cc63e16adfeed879890449df8394b20840d Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Thu, 26 Jan 2023 23:19:51 +0300 Subject: [PATCH 58/90] scaladoc #2 --- .../org/ergoplatform/nodeView/ErgoNodeViewHolder.scala | 6 +++++- src/main/scala/scorex/core/network/DeliveryTracker.scala | 3 +++ .../scorex/core/network/message/BasicMessagesRepo.scala | 3 +++ src/main/scala/scorex/core/validation/ModifierError.scala | 4 ++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index 954566ccef..9685a99fe8 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -133,7 +133,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti } - protected def requestDownloads(pi: ProgressInfo[BlockSection]): Unit = { + private def requestDownloads(pi: ProgressInfo[BlockSection]): Unit = { //TODO: actually, pi.toDownload contains only 1 modifierid per type, //TODO: see the only case where toDownload is not empty during ProgressInfo construction //TODO: so the code below can be optimized @@ -720,6 +720,10 @@ object ErgoNodeViewHolder { case class BlockAppliedTransactions(txs: Seq[ModifierId]) extends NodeViewHolderEvent + /** + * When node view holder is realizing it knows IDs of block sections not downloaded yte, it sends this signal + * to download them + */ case class DownloadRequest(modifiersToFetch: Map[NetworkObjectTypeId.Value, Seq[ModifierId]]) extends NodeViewHolderEvent case class CurrentView[State](history: ErgoHistory, state: State, vault: ErgoWallet, pool: ErgoMemPool) diff --git a/src/main/scala/scorex/core/network/DeliveryTracker.scala b/src/main/scala/scorex/core/network/DeliveryTracker.scala index b6231445a5..43ecf16889 100644 --- a/src/main/scala/scorex/core/network/DeliveryTracker.scala +++ b/src/main/scala/scorex/core/network/DeliveryTracker.scala @@ -253,6 +253,9 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, case _ => false } + /** + * Clear old known status `oldStatus` from tracked block section `modifierId` of type `modifierTypeId` + */ def clearStatusForModifier(id: ModifierId, modifierTypeId: NetworkObjectTypeId.Value, oldStatus: ModifiersStatus): Unit = oldStatus match { case Requested => diff --git a/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala b/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala index 68e5f0364f..396a63c676 100644 --- a/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala +++ b/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala @@ -13,6 +13,9 @@ import scorex.util.{ModifierId, ScorexLogging, bytesToId, idToBytes} import scala.collection.immutable +/** + * Wrapper for block sections of the same type. Used to send multiple block sections at once ove the wire. + */ case class ModifiersData(typeId: NetworkObjectTypeId.Value, modifiers: Map[ModifierId, Array[Byte]]) case class InvData(typeId: NetworkObjectTypeId.Value, ids: Seq[ModifierId]) diff --git a/src/main/scala/scorex/core/validation/ModifierError.scala b/src/main/scala/scorex/core/validation/ModifierError.scala index 672c93db8f..a80444668a 100644 --- a/src/main/scala/scorex/core/validation/ModifierError.scala +++ b/src/main/scala/scorex/core/validation/ModifierError.scala @@ -5,6 +5,10 @@ import scorex.util.ModifierId import scala.util.control.NoStackTrace +/** + * Container for error details in regards with block section turned out to be invalid. Wraps validation error, + * block section id, and block section type id. + */ case class InvalidModifier(error: String, modifierId: ModifierId, modifierTypeId: NetworkObjectTypeId.Value) /** Base trait for errors that were occurred during NodeView Modifier validation From aa47d137185555357e6dfe10ba6b3f2df129ef56 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Sat, 28 Jan 2023 03:49:41 +0300 Subject: [PATCH 59/90] ntp dependency removed --- .../org/ergoplatform/bench/BenchRunner.scala | 9 +-- .../ergoplatform/bench/HistoryExtractor.scala | 7 +-- build.sbt | 1 - src/main/resources/application.conf | 10 ---- src/main/scala/org/ergoplatform/ErgoApp.scala | 16 ++--- .../ergoplatform/http/api/InfoApiRoute.scala | 6 +- .../local/ErgoStatsCollector.scala | 21 +++---- .../mining/CandidateGenerator.scala | 10 +--- .../org/ergoplatform/mining/ErgoMiner.scala | 8 +-- .../modifiers/history/header/Header.scala | 5 +- .../network/ErgoNodeViewSynchronizer.scala | 13 ++-- .../ergoplatform/network/ErgoPeerStatus.scala | 2 +- .../network/ErgoSyncTracker.scala | 25 ++++---- .../nodeView/ErgoNodeViewHolder.scala | 47 +++++++-------- .../nodeView/history/ErgoHistory.scala | 9 +-- .../modifierprocessors/HeadersProcessor.scala | 6 +- .../ToDownloadProcessor.scala | 5 +- .../tools/DifficultyEstimation.scala | 10 +--- .../scala/scorex/core/app/Application.scala | 4 -- .../scala/scorex/core/app/ScorexContext.scala | 2 - .../core/network/NetworkController.scala | 21 +++---- .../core/network/PeerConnectionHandler.scala | 4 +- .../core/network/peer/PeerDatabase.scala | 15 ++--- .../core/network/peer/PeerManager.scala | 2 +- .../scala/scorex/core/settings/Settings.scala | 4 +- .../scala/scorex/core/utils/NetworkTime.scala | 59 ------------------- .../scorex/core/utils/TimeProvider.scala | 9 --- .../http/routes/ErgoPeersApiRouteSpec.scala | 6 -- .../http/routes/InfoApiRoutesSpec.scala | 12 +--- .../mining/CandidateGeneratorSpec.scala | 12 ++-- .../ergoplatform/mining/ErgoMinerSpec.scala | 18 ++---- ...rgoNodeViewSynchronizerSpecification.scala | 28 ++++----- .../ErgoSyncTrackerSpecification.scala | 2 +- .../viewholder/ErgoNodeViewHolderSpec.scala | 4 +- .../org/ergoplatform/sanity/ErgoSanity.scala | 11 ++-- .../sanity/ErgoSanityDigest.scala | 5 +- .../ergoplatform/sanity/ErgoSanityUTXO.scala | 7 +-- .../ergoplatform/tools/ChainGenerator.scala | 6 +- .../utils/ErgoTestConstants.scala | 2 - .../ergoplatform/utils/ErgoTestHelpers.scala | 10 +--- .../utils/HistoryTestHelpers.scala | 2 +- .../ergoplatform/utils/NodeViewTestOps.scala | 2 +- .../scala/org/ergoplatform/utils/Stubs.scala | 6 +- .../utils/fixtures/NodeViewFixture.scala | 8 +-- .../utils/generators/ChainGenerator.scala | 4 +- .../generators/ValidBlocksGenerators.scala | 4 +- 46 files changed, 143 insertions(+), 336 deletions(-) delete mode 100644 src/main/scala/scorex/core/utils/NetworkTime.scala delete mode 100644 src/main/scala/scorex/core/utils/TimeProvider.scala diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/BenchRunner.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/BenchRunner.scala index c8fbcbc157..2dfdfbfd0e 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/bench/BenchRunner.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/bench/BenchRunner.scala @@ -10,12 +10,8 @@ import org.ergoplatform.nodeView.{ErgoNodeViewRef, NVBenchmark} import org.ergoplatform.settings.{Args, ErgoSettings} import org.ergoplatform.nodeView.ErgoNodeViewHolder.CurrentView import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{GetDataFromCurrentView, LocallyGeneratedModifier} -import scorex.core.utils.{NetworkTimeProvider, NetworkTimeProviderSettings} import scorex.util.ScorexLogging - import scala.concurrent.ExecutionContextExecutor -import scala.concurrent.duration._ -import scala.language.postfixOps object BenchRunner extends ScorexLogging with NVBenchmark { @@ -41,10 +37,7 @@ object BenchRunner extends ScorexLogging with NVBenchmark { log.info(s"Setting that being used:") log.info(s"$ergoSettings") - val ntpSettings = NetworkTimeProviderSettings("pool.ntp.org", 30 minutes, 30 seconds) - val timeProvider = new NetworkTimeProvider(ntpSettings) - - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) /** * It's a hack to set minimalFullBlockHeightVar to 0 and to avoid "Header Is Not Synced" error, cause diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala index 027bafa170..27c17a2c1f 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/bench/HistoryExtractor.scala @@ -5,11 +5,8 @@ import java.io.FileOutputStream import org.ergoplatform.bench.misc.ModifierWriter import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.settings.{Args, ErgoSettings} -import scorex.core.settings.ScorexSettings -import scorex.core.utils.NetworkTimeProvider import scorex.util.ScorexLogging -import scala.concurrent.ExecutionContext.Implicits.global object HistoryExtractor extends ScorexLogging { @@ -18,11 +15,9 @@ object HistoryExtractor extends ScorexLogging { lazy val cfgPath: Option[String] = args.headOption lazy val outputFile: String = args.lift(1).getOrElse("blocks.dat") lazy val ergoSettings: ErgoSettings = ErgoSettings.read(Args(cfgPath, None)) - lazy val settings: ScorexSettings = ergoSettings.scorexSettings - val timeProvider = new NetworkTimeProvider(settings.ntp) val os = new FileOutputStream(outputFile) - val h = ErgoHistory.readOrGenerate(ergoSettings, timeProvider) + val h = ErgoHistory.readOrGenerate(ergoSettings) val wholeChain = h.chainToHeader(None, h.bestHeaderOpt.get) var counter = 0 diff --git a/build.sbt b/build.sbt index 41806f983f..7a7dcc9d12 100644 --- a/build.sbt +++ b/build.sbt @@ -61,7 +61,6 @@ libraryDependencies ++= Seq( "com.typesafe.akka" %% "akka-parsing" % akkaHttpVersion, "com.typesafe.akka" %% "akka-stream" % akkaVersion, "org.bitlet" % "weupnp" % "0.1.4", - "commons-net" % "commons-net" % "3.6", // api dependencies "io.circe" %% "circe-core" % circeVersion, diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 319ad12a33..7eebce2f6e 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -539,16 +539,6 @@ scorex { peerDiscovery = true } - ntp { - # NTP server address - server = "pool.ntp.org" - - # update time rate - updateEvery = 30m - - # server answer timeout - timeout = 30s - } } diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index 54d26d41ed..c759ba6462 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -21,7 +21,6 @@ import scorex.core.network.message.Message.MessageCode import scorex.core.network.message._ import scorex.core.network.peer.PeerManagerRef import scorex.core.settings.ScorexSettings -import scorex.core.utils.NetworkTimeProvider import scorex.util.ScorexLogging import java.net.InetSocketAddress @@ -50,8 +49,6 @@ class ErgoApp(args: Args) extends ScorexLogging { implicit private val executionContext: ExecutionContext = actorSystem.dispatcher - private val timeProvider = new NetworkTimeProvider(scorexSettings.ntp) - private val upnpGateway: Option[UPnPGateway] = if (scorexSettings.network.upnpEnabled) UPnP.getValidGateway(scorexSettings.network) else None @@ -80,25 +77,24 @@ class ErgoApp(args: Args) extends ScorexLogging { private val scorexContext = ScorexContext( messageSpecs = basicSpecs ++ additionalMessageSpecs, upnpGateway = upnpGateway, - timeProvider = timeProvider, externalNodeAddress = externalSocketAddress ) private val peerManagerRef = PeerManagerRef(ergoSettings, scorexContext) - private val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + private val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) private val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) // Create an instance of ErgoMiner actor if "mining = true" in config private val minerRefOpt: Option[ActorRef] = if (ergoSettings.nodeSettings.mining) { - Some(ErgoMiner(ergoSettings, nodeViewHolderRef, readersHolderRef, timeProvider)) + Some(ErgoMiner(ergoSettings, nodeViewHolderRef, readersHolderRef)) } else { None } - private val syncTracker = ErgoSyncTracker(scorexSettings.network, timeProvider) + private val syncTracker = ErgoSyncTracker(scorexSettings.network) private val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(ergoSettings) @@ -107,7 +103,6 @@ class ErgoApp(args: Args) extends ScorexLogging { nodeViewHolderRef, ErgoSyncInfoMessageSpec, ergoSettings, - timeProvider, syncTracker, deliveryTracker ) @@ -152,8 +147,7 @@ class ErgoApp(args: Args) extends ScorexLogging { readersHolderRef, networkControllerRef, syncTracker, - ergoSettings, - timeProvider + ergoSettings ) private val apiRoutes: Seq[ApiRoute] = Seq( @@ -166,7 +160,7 @@ class ErgoApp(args: Args) extends ScorexLogging { deliveryTracker, scorexSettings.restApi ), - InfoApiRoute(statsCollectorRef, scorexSettings.restApi, timeProvider), + InfoApiRoute(statsCollectorRef, scorexSettings.restApi), BlocksApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings), NipopowApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings), TransactionsApiRoute(readersHolderRef, nodeViewHolderRef, ergoSettings), diff --git a/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala index e583790be6..ad7835f59b 100644 --- a/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala @@ -7,16 +7,14 @@ import io.circe.syntax._ import org.ergoplatform.local.ErgoStatsCollector.{GetNodeInfo, NodeInfo} import scorex.core.api.http.ApiResponse import scorex.core.settings.RESTApiSettings -import scorex.core.utils.NetworkTimeProvider case class InfoApiRoute(statsCollector: ActorRef, - settings: RESTApiSettings, - timeProvider: NetworkTimeProvider) + settings: RESTApiSettings) (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute { override val route: Route = (path("info") & get) { - val timeJson = Map("currentTime" -> timeProvider.time().asJson).asJson + val timeJson = Map("currentTime" -> System.currentTimeMillis().asJson).asJson ApiResponse((statsCollector ? GetNodeInfo).mapTo[NodeInfo].map(_.asJson.deepMerge(timeJson))) } diff --git a/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala b/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala index bb41748619..60026d4516 100644 --- a/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala +++ b/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala @@ -16,8 +16,6 @@ import scorex.core.network.ConnectedPeer import scorex.core.network.NetworkController.ReceivableMessages.{GetConnectedPeers, GetPeersStatus} import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages._ import org.ergoplatform.network.ErgoSyncTracker -import scorex.core.utils.NetworkTimeProvider -import scorex.core.utils.TimeProvider.Time import scorex.util.ScorexLogging import scorex.core.network.peer.PeersStatus @@ -31,8 +29,7 @@ import scala.concurrent.duration._ class ErgoStatsCollector(readersHolder: ActorRef, networkController: ActorRef, syncTracker: ErgoSyncTracker, - settings: ErgoSettings, - timeProvider: NetworkTimeProvider) + settings: ErgoSettings) extends Actor with ScorexLogging { override def preStart(): Unit = { @@ -47,8 +44,6 @@ class ErgoStatsCollector(readersHolder: ActorRef, context.system.scheduler.scheduleAtFixedRate(45.seconds, 30.seconds, networkController, GetPeersStatus)(ec, self) } - private def networkTime(): Time = timeProvider.time() - private var nodeInfo = NodeInfo( settings.scorexSettings.network.nodeName, Version.VersionString, @@ -64,8 +59,8 @@ class ErgoStatsCollector(readersHolder: ActorRef, None, None, None, - launchTime = networkTime(), - lastIncomingMessageTime = networkTime(), + launchTime = System.currentTimeMillis(), + lastIncomingMessageTime = System.currentTimeMillis(), None, LaunchParameters, eip27Supported = true, @@ -241,16 +236,14 @@ object ErgoStatsCollectorRef { def props(readersHolder: ActorRef, networkController: ActorRef, syncTracker : ErgoSyncTracker, - settings: ErgoSettings, - timeProvider: NetworkTimeProvider): Props = - Props(new ErgoStatsCollector(readersHolder, networkController, syncTracker, settings, timeProvider)) + settings: ErgoSettings): Props = + Props(new ErgoStatsCollector(readersHolder, networkController, syncTracker, settings)) def apply(readersHolder: ActorRef, networkController: ActorRef, syncTracker : ErgoSyncTracker, - settings: ErgoSettings, - timeProvider: NetworkTimeProvider)(implicit system: ActorSystem): ActorRef = - system.actorOf(props(readersHolder, networkController, syncTracker, settings, timeProvider)) + settings: ErgoSettings)(implicit system: ActorSystem): ActorRef = + system.actorOf(props(readersHolder, networkController, syncTracker, settings)) } diff --git a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala index a24dd82c22..dbdab5b7f2 100644 --- a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala +++ b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala @@ -24,7 +24,6 @@ import org.ergoplatform.wallet.Constants.MaxAssetsPerBox import org.ergoplatform.wallet.interpreter.ErgoInterpreter import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoScriptPredef, Input} import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{EliminateTransactions, LocallyGeneratedModifier} -import scorex.core.utils.NetworkTimeProvider import scorex.crypto.hash.Digest32 import scorex.util.encode.Base16 import scorex.util.{ModifierId, ScorexLogging} @@ -46,7 +45,6 @@ class CandidateGenerator( minerPk: ProveDlog, readersHolderRef: ActorRef, viewHolderRef: ActorRef, - timeProvider: NetworkTimeProvider, ergoSettings: ErgoSettings ) extends Actor with ScorexLogging { @@ -147,7 +145,6 @@ class CandidateGenerator( state.hr, state.sr, state.mpr, - timeProvider, minerPk, txsToInclude, ergoSettings @@ -256,7 +253,6 @@ object CandidateGenerator extends ScorexLogging { minerPk: ProveDlog, readersHolderRef: ActorRef, viewHolderRef: ActorRef, - timeProvider: NetworkTimeProvider, ergoSettings: ErgoSettings )(implicit context: ActorRefFactory): ActorRef = context.actorOf( @@ -265,7 +261,6 @@ object CandidateGenerator extends ScorexLogging { minerPk, readersHolderRef, viewHolderRef, - timeProvider, ergoSettings ) ).withDispatcher("critical-dispatcher"), @@ -328,7 +323,6 @@ object CandidateGenerator extends ScorexLogging { h: ErgoHistoryReader, s: UtxoStateReader, m: ErgoMemPoolReader, - timeProvider: NetworkTimeProvider, pk: ProveDlog, txsToInclude: Seq[ErgoTransaction], ergoSettings: ErgoSettings @@ -368,7 +362,6 @@ object CandidateGenerator extends ScorexLogging { h, desiredUpdate, s, - timeProvider, poolTransactions, emissionTxOpt, unspentTxsToInclude, @@ -445,7 +438,6 @@ object CandidateGenerator extends ScorexLogging { history: ErgoHistoryReader, proposedUpdate: ErgoValidationSettingsUpdate, state: UtxoStateReader, - timeProvider: NetworkTimeProvider, poolTxs: Seq[UnconfirmedTransaction], emissionTxOpt: Option[ErgoTransaction], prioritizedTransactions: Seq[ErgoTransaction], @@ -461,7 +453,7 @@ object CandidateGenerator extends ScorexLogging { // Make progress in time since last block. // If no progress is made, then, by consensus rules, the block will be rejected. val timestamp = - Math.max(timeProvider.time(), bestHeaderOpt.map(_.timestamp + 1).getOrElse(0L)) + Math.max(System.currentTimeMillis(), bestHeaderOpt.map(_.timestamp + 1).getOrElse(0L)) val stateContext = state.stateContext diff --git a/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala b/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala index fef6d1eb0f..887845cd20 100644 --- a/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala +++ b/src/main/scala/org/ergoplatform/mining/ErgoMiner.scala @@ -9,7 +9,6 @@ import org.ergoplatform.nodeView.wallet.ErgoWalletActor.{FirstSecretResponse, Ge import org.ergoplatform.settings.ErgoSettings import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.GetDataFromCurrentView import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.FullBlockApplied -import scorex.core.utils.NetworkTimeProvider import scorex.util.ScorexLogging import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog} @@ -24,7 +23,6 @@ class ErgoMiner( ergoSettings: ErgoSettings, viewHolderRef: ActorRef, readersHolderRef: ActorRef, - timeProvider: NetworkTimeProvider, secretKeyOpt: Option[DLogProverInput] ) extends Actor with Stash @@ -62,7 +60,6 @@ class ErgoMiner( publicKey, readersHolderRef, viewHolderRef, - timeProvider, ergoSettings ) context.become(starting(MinerState(secretKeyOpt, publicKey, candidateGeneratorRef))) @@ -116,7 +113,7 @@ class ErgoMiner( /** We need to ignore all historical blocks, mining is triggered by latest blocks only */ private def shouldStartMine(b: Header): Boolean = { - b.isNew(timeProvider, ergoSettings.chainSettings.blockInterval * 2) + b.isNew(ergoSettings.chainSettings.blockInterval * 2) } /** Let's wait for a signal to start mining, either from ErgoApp or when a latest blocks get applied to blockchain */ @@ -201,12 +198,11 @@ object ErgoMiner extends ScorexLogging { ergoSettings: ErgoSettings, viewHolderRef: ActorRef, readersHolderRef: ActorRef, - timeProvider: NetworkTimeProvider, skOpt: Option[DLogProverInput] = None )(implicit context: ActorRefFactory): ActorRef = context.actorOf( Props( - new ErgoMiner(ergoSettings, viewHolderRef, readersHolderRef, timeProvider, skOpt) + new ErgoMiner(ergoSettings, viewHolderRef, readersHolderRef, skOpt) ) ) } diff --git a/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala b/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala index 58919ee96d..108d3a215c 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala @@ -13,7 +13,6 @@ import org.ergoplatform.nodeView.history.ErgoHistory.Difficulty import org.ergoplatform.settings.{Algos, Constants} import org.ergoplatform.wallet.interpreter.ErgoInterpreter import scorex.core.serialization.ScorexSerializer -import scorex.core.utils.NetworkTimeProvider import scorex.core.ModifierTypeId import scorex.crypto.authds.ADDigest import scorex.crypto.hash.Digest32 @@ -102,8 +101,8 @@ case class Header(override val version: Header.Version, /** * Estimate that this header is recent enough to possibly be the best header */ - def isNew(timeProvider: NetworkTimeProvider, timeDiff: FiniteDuration): Boolean = { - timeProvider.time() - timestamp < timeDiff.toMillis + def isNew(timeDiff: FiniteDuration): Boolean = { + System.currentTimeMillis() - timestamp < timeDiff.toMillis } /** diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 7a94a60ec5..fcc0db78de 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -27,7 +27,7 @@ import scorex.core.network.message.{InvData, Message, ModifiersData} import scorex.core.serialization.ScorexSerializer import scorex.core.settings.NetworkSettings import scorex.core.transaction.Transaction -import scorex.core.utils.{NetworkTimeProvider, ScorexEncoding} +import scorex.core.utils.ScorexEncoding import scorex.core.validation.MalformedModifierError import scorex.util.{ModifierId, ScorexLogging} import scorex.core.network.DeliveryTracker @@ -49,7 +49,6 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, - timeProvider: NetworkTimeProvider, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker )(implicit ex: ExecutionContext) @@ -1038,7 +1037,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, // If new enough semantically valid ErgoFullBlock was applied, send inv for block header and all its sections case FullBlockApplied(header) => - if (header.isNew(timeProvider, 2.hours)) { + if (header.isNew(2.hours)) { broadcastModifierInv(Header.modifierTypeId, header.id) header.sectionIds.foreach { case (mtId, id) => broadcastModifierInv(mtId, id) } } @@ -1188,21 +1187,19 @@ object ErgoNodeViewSynchronizer { viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, - timeProvider: NetworkTimeProvider, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker) (implicit ex: ExecutionContext): Props = - Props(new ErgoNodeViewSynchronizer(networkControllerRef, viewHolderRef, syncInfoSpec, settings, - timeProvider, syncTracker, deliveryTracker)) + Props(new ErgoNodeViewSynchronizer(networkControllerRef, viewHolderRef, syncInfoSpec, + settings, syncTracker, deliveryTracker)) def make(viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, - timeProvider: NetworkTimeProvider, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker) (implicit context: ActorRefFactory, ex: ExecutionContext): ActorRef => ActorRef = - networkControllerRef => context.actorOf(props(networkControllerRef, viewHolderRef, syncInfoSpec, settings, timeProvider, syncTracker, deliveryTracker)) + networkControllerRef => context.actorOf(props(networkControllerRef, viewHolderRef, syncInfoSpec, settings, syncTracker, deliveryTracker)) /** * Container for aggregated costs of accepted, declined or invalidated transactions. Can be used to track global diff --git a/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala b/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala index cef1acd5a8..4d5269819c 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala @@ -5,7 +5,7 @@ import org.ergoplatform.nodeView.history.ErgoHistory.Height import scorex.core.app.Version import scorex.core.consensus.PeerChainStatus import scorex.core.network.ConnectedPeer -import scorex.core.utils.TimeProvider.Time +import org.ergoplatform.nodeView.history.ErgoHistory.Time /** * Container for status of another peer diff --git a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala index 3c0fcec684..6783f5fdab 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala @@ -3,17 +3,16 @@ package org.ergoplatform.network import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader, ErgoSyncInfo, ErgoSyncInfoV1, ErgoSyncInfoV2} import org.ergoplatform.nodeView.history.ErgoHistory.Height -import scorex.core.consensus.{Fork, PeerChainStatus, Older, Unknown} +import scorex.core.consensus.{Fork, Older, PeerChainStatus, Unknown} import scorex.core.network.ConnectedPeer import scorex.core.settings.NetworkSettings -import scorex.core.utils.TimeProvider import scorex.util.ScorexLogging import scala.collection.mutable import scala.concurrent.duration._ import scorex.core.utils.MapPimp -final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: TimeProvider) extends ScorexLogging { +final case class ErgoSyncTracker(networkSettings: NetworkSettings) extends ScorexLogging { private val MinSyncInterval: FiniteDuration = 20.seconds private val SyncThreshold: FiniteDuration = 1.minute @@ -27,12 +26,14 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: def fullInfo(): Iterable[ErgoPeerStatus] = statuses.values + private def currentTime() = System.currentTimeMillis() + // returns diff def updateLastSyncGetTime(peer: ConnectedPeer): Long = { val prevSyncGetTime = statuses.get(peer).flatMap(_.lastSyncGetTime).getOrElse(0L) - val currentTime = timeProvider.time() + statuses.get(peer).foreach { status => - statuses.update(peer, status.copy(lastSyncGetTime = Option(currentTime))) + statuses.update(peer, status.copy(lastSyncGetTime = Option(currentTime()))) } currentTime - prevSyncGetTime } @@ -43,7 +44,7 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: val outdated = peerOpt .flatMap(_.lastSyncSentTime) - .exists(syncTime => (timeProvider.time() - syncTime).millis > SyncThreshold) + .exists(syncTime => (System.currentTimeMillis() - syncTime).millis > SyncThreshold) notSyncedOrMissing || outdated } @@ -108,9 +109,8 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: } def updateLastSyncSentTime(peer: ConnectedPeer): Unit = { - val currentTime = timeProvider.time() statuses.get(peer).foreach { status => - statuses.update(peer, status.copy(lastSyncSentTime = Option(currentTime))) + statuses.update(peer, status.copy(lastSyncSentTime = Option(currentTime()))) } } @@ -118,9 +118,8 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: * Helper method to clear statuses of peers not updated for long enough */ private[network] def clearOldStatuses(): Unit = { - val currentTime = timeProvider.time() val peersToClear = statuses.filter { case (_, status) => - status.lastSyncSentTime.exists(syncTime => (currentTime - syncTime).millis > ClearThreshold) + status.lastSyncSentTime.exists(syncTime => (currentTime() - syncTime).millis > ClearThreshold) }.keys if (peersToClear.nonEmpty) { log.debug(s"Clearing stalled statuses for $peersToClear") @@ -130,9 +129,8 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: } private[network] def outdatedPeers: IndexedSeq[ConnectedPeer] = { - val currentTime = timeProvider.time() statuses.filter { case (_, status) => - status.lastSyncSentTime.exists(syncTime => (currentTime - syncTime).millis > SyncThreshold) + status.lastSyncSentTime.exists(syncTime => (currentTime() - syncTime).millis > SyncThreshold) }.keys.toVector } @@ -168,7 +166,6 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: if (outdated.nonEmpty) { outdated } else { - val currentTime = timeProvider.time() val unknowns = statuses.filter(_._2.status == Unknown).toVector val forks = statuses.filter(_._2.status == Fork).toVector val elders = statuses.filter(_._2.status == Older).toVector @@ -180,7 +177,7 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings, timeProvider: } val nonOutdated = eldersAndUnknown ++ forks nonOutdated.filter { case (_, status) => - (currentTime - status.lastSyncSentTime.getOrElse(0L)).millis >= MinSyncInterval + (currentTime() - status.lastSyncSentTime.getOrElse(0L)).millis >= MinSyncInterval }.map(_._1) } diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index 326f63052f..54a4521f55 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -22,7 +22,7 @@ import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages._ import org.ergoplatform.modifiers.history.{ADProofs, HistoryModifierSerializer} import scorex.core.consensus.ProgressInfo import scorex.core.settings.ScorexSettings -import scorex.core.utils.{NetworkTimeProvider, ScorexEncoding} +import scorex.core.utils.ScorexEncoding import scorex.core.validation.RecoverableModifierError import scorex.util.{ModifierId, ScorexLogging} import spire.syntax.all.cfor @@ -39,8 +39,7 @@ import scala.util.{Failure, Success, Try} * Updates them atomically. * */ -abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSettings, - timeProvider: NetworkTimeProvider) +abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSettings) extends Actor with ScorexLogging with ScorexEncoding with FileUtils { private implicit lazy val actorSystem: ActorSystem = context.system @@ -385,7 +384,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti val state = recreatedState() - val history = ErgoHistory.readOrGenerate(settings, timeProvider) + val history = ErgoHistory.readOrGenerate(settings) val wallet = ErgoWallet.readOrGenerate( history.getReader, settings, LaunchParameters) @@ -403,7 +402,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti def restoreState(): Option[NodeView] = if (ErgoHistory.historyDir(settings).listFiles().isEmpty) { None } else { - val history = ErgoHistory.readOrGenerate(settings, timeProvider) + val history = ErgoHistory.readOrGenerate(settings) log.info("History database read") val memPool = ErgoMemPool.empty(settings) val constants = StateConstants(settings) @@ -496,7 +495,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti log.info(s"Persistent modifier ${pmod.encodedId} applied successfully") updateNodeView(Some(newHistory), Some(newMinState), Some(newVault), Some(newMemPool)) chainProgress = - Some(ChainProgress(pmod, headersHeight, fullBlockHeight, timeProvider.time())) + Some(ChainProgress(pmod, headersHeight, fullBlockHeight, System.currentTimeMillis())) case Failure(e) => log.warn(s"Can`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod) to minimal state", e) updateNodeView(updatedHistory = Some(newHistory)) @@ -656,7 +655,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti case IsChainHealthy => log.info(s"Check that chain is healthy, progress is $chainProgress") val healthCheckReply = chainProgress.map { progress => - ErgoNodeViewHolder.checkChainIsHealthy(progress, history(), timeProvider, settings) + ErgoNodeViewHolder.checkChainIsHealthy(progress, history(), settings) }.getOrElse(ChainIsStuck("Node already stuck when started")) sender() ! healthCheckReply } @@ -729,10 +728,9 @@ object ErgoNodeViewHolder { def checkChainIsHealthy( progress: ChainProgress, history: ErgoHistory, - timeProvider: NetworkTimeProvider, settings: ErgoSettings): HealthCheckResult = { val ChainProgress(lastMod, headersHeight, blockHeight, lastUpdate) = progress - val chainUpdateDelay = timeProvider.time() - lastUpdate + val chainUpdateDelay = System.currentTimeMillis() - lastUpdate val acceptableChainUpdateDelay = settings.nodeSettings.acceptableChainUpdateDelay def chainUpdateDelayed = chainUpdateDelay > acceptableChainUpdateDelay.toMillis def chainSynced = @@ -754,34 +752,29 @@ object ErgoNodeViewHolder { } } -private[nodeView] class DigestNodeViewHolder(settings: ErgoSettings, - timeProvider: NetworkTimeProvider) - extends ErgoNodeViewHolder[DigestState](settings, timeProvider) +private[nodeView] class DigestNodeViewHolder(settings: ErgoSettings) + extends ErgoNodeViewHolder[DigestState](settings) -private[nodeView] class UtxoNodeViewHolder(settings: ErgoSettings, - timeProvider: NetworkTimeProvider) - extends ErgoNodeViewHolder[UtxoState](settings, timeProvider) +private[nodeView] class UtxoNodeViewHolder(settings: ErgoSettings) + extends ErgoNodeViewHolder[UtxoState](settings) object ErgoNodeViewRef { - private def digestProps(settings: ErgoSettings, - timeProvider: NetworkTimeProvider): Props = - Props.create(classOf[DigestNodeViewHolder], settings, timeProvider) + private def digestProps(settings: ErgoSettings): Props = + Props.create(classOf[DigestNodeViewHolder], settings) - private def utxoProps(settings: ErgoSettings, - timeProvider: NetworkTimeProvider): Props = - Props.create(classOf[UtxoNodeViewHolder], settings, timeProvider) + private def utxoProps(settings: ErgoSettings): Props = + Props.create(classOf[UtxoNodeViewHolder], settings) - def props(settings: ErgoSettings, - timeProvider: NetworkTimeProvider): Props = + def props(settings: ErgoSettings): Props = (settings.nodeSettings.stateType match { - case StateType.Digest => digestProps(settings, timeProvider) - case StateType.Utxo => utxoProps(settings, timeProvider) + case StateType.Digest => digestProps(settings) + case StateType.Utxo => utxoProps(settings) }).withDispatcher("critical-dispatcher") - def apply(settings: ErgoSettings, timeProvider: NetworkTimeProvider)(implicit system: ActorSystem): ActorRef = - system.actorOf(props(settings, timeProvider)) + def apply(settings: ErgoSettings)(implicit system: ActorSystem): ActorRef = + system.actorOf(props(settings)) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala index e9f3f151aa..d153c58c48 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala @@ -12,7 +12,6 @@ import org.ergoplatform.nodeView.history.storage.modifierprocessors._ import org.ergoplatform.settings._ import org.ergoplatform.utils.LoggingUtil import scorex.core.consensus.ProgressInfo -import scorex.core.utils.NetworkTimeProvider import scorex.core.validation.RecoverableModifierError import scorex.util.{ModifierId, ScorexLogging, idToBytes} @@ -232,6 +231,8 @@ trait ErgoHistory object ErgoHistory extends ScorexLogging { + type Time = Long + type Height = ErgoLikeContext.Height // Int type Score = BigInt type Difficulty = BigInt @@ -269,7 +270,7 @@ object ErgoHistory extends ScorexLogging { } } - def readOrGenerate(ergoSettings: ErgoSettings, ntp: NetworkTimeProvider): ErgoHistory = { + def readOrGenerate(ergoSettings: ErgoSettings): ErgoHistory = { val db = HistoryStorage(ergoSettings) val nodeSettings = ergoSettings.nodeSettings @@ -279,7 +280,6 @@ object ErgoHistory extends ScorexLogging { override protected val settings: ErgoSettings = ergoSettings override protected[history] val historyStorage: HistoryStorage = db override val powScheme: AutolykosPowScheme = chainSettings.powScheme - override protected val timeProvider: NetworkTimeProvider = ntp } case (false, true) => @@ -287,7 +287,6 @@ object ErgoHistory extends ScorexLogging { override protected val settings: ErgoSettings = ergoSettings override protected[history] val historyStorage: HistoryStorage = db override val powScheme: AutolykosPowScheme = chainSettings.powScheme - override protected val timeProvider: NetworkTimeProvider = ntp } case (true, false) => @@ -295,7 +294,6 @@ object ErgoHistory extends ScorexLogging { override protected val settings: ErgoSettings = ergoSettings override protected[history] val historyStorage: HistoryStorage = db override val powScheme: AutolykosPowScheme = chainSettings.powScheme - override protected val timeProvider: NetworkTimeProvider = ntp } case (false, false) => @@ -303,7 +301,6 @@ object ErgoHistory extends ScorexLogging { override protected val settings: ErgoSettings = ergoSettings override protected[history] val historyStorage: HistoryStorage = db override val powScheme: AutolykosPowScheme = chainSettings.powScheme - override protected val timeProvider: NetworkTimeProvider = ntp } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala index 36010fc66d..ddab1a8582 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala @@ -311,6 +311,8 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score private def validationState: ValidationState[Unit] = ModifierValidator(ErgoValidationSettings.initial) + private def time() = System.currentTimeMillis() + def validate(header: Header): ValidationResult[Unit] = { if (header.isGenesis) { validateGenesisBlockHeader(header) @@ -333,7 +335,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score .validateEquals(hdrRequiredDifficulty, header.requiredDifficulty, chainSettings.initialDifficulty, header.id, header.modifierTypeId) .validateNot(alreadyApplied, historyStorage.contains(header.id), InvalidModifier(header.toString, header.id, header.modifierTypeId)) .validate(hdrTooOld, fullBlockHeight < nodeSettings.keepVersions, InvalidModifier(heightOf(header.parentId).toString, header.id, header.modifierTypeId)) - .validate(hdrFutureTimestamp, header.timestamp - timeProvider.time() <= MaxTimeDrift, InvalidModifier(s"${header.timestamp} vs ${timeProvider.time()}", header.id, header.modifierTypeId)) + .validate(hdrFutureTimestamp, header.timestamp - time() <= MaxTimeDrift, InvalidModifier(s"${header.timestamp} vs ${time()}", header.id, header.modifierTypeId)) .result } @@ -348,7 +350,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score .validateEquals(hdrRequiredDifficulty, header.requiredDifficulty, requiredDifficultyAfter(parent), header.id, header.modifierTypeId) .validate(hdrTooOld, heightOf(header.parentId).exists(h => fullBlockHeight - h < nodeSettings.keepVersions), InvalidModifier(heightOf(header.parentId).toString, header.id, header.modifierTypeId)) .validateSemantics(hdrParentSemantics, isSemanticallyValid(header.parentId), InvalidModifier(s"Parent semantics broken", header.id, header.modifierTypeId)) - .validate(hdrFutureTimestamp, header.timestamp - timeProvider.time() <= MaxTimeDrift, InvalidModifier(s"${header.timestamp} vs ${timeProvider.time()}", header.id, header.modifierTypeId)) + .validate(hdrFutureTimestamp, header.timestamp - time() <= MaxTimeDrift, InvalidModifier(s"${header.timestamp} vs ${time()}", header.id, header.modifierTypeId)) .validateNot(alreadyApplied, historyStorage.contains(header.id), InvalidModifier(s"${header.id} already applied", header.id, header.modifierTypeId)) .validate(hdrCheckpoint, checkpointCondition(header), InvalidModifier(s"${header.id} wrong checkpoint", header.id, header.modifierTypeId)) .result diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala index 97af3cc922..310302160c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala @@ -5,7 +5,6 @@ import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.settings.{ChainSettings, ErgoSettings, NodeConfigurationSettings} import scorex.core.ModifierTypeId -import scorex.core.utils.NetworkTimeProvider import scorex.util.{ModifierId, ScorexLogging} import scala.annotation.tailrec @@ -16,8 +15,6 @@ import scala.annotation.tailrec trait ToDownloadProcessor extends BasicReaders with ScorexLogging { import ToDownloadProcessor._ - protected val timeProvider: NetworkTimeProvider - protected val settings: ErgoSettings // A node is considering that the chain is synced if sees a block header with timestamp no more @@ -107,7 +104,7 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { } else if (pruningProcessor.shouldDownloadBlockAtHeight(header.height)) { // Already synced and header is not too far back. Download required modifiers. requiredModifiersForHeader(header) - } else if (!isHeadersChainSynced && header.isNew(timeProvider, chainSettings.blockInterval * headerChainDiff)) { + } else if (!isHeadersChainSynced && header.isNew(chainSettings.blockInterval * headerChainDiff)) { // Headers chain is synced after this header. Start downloading full blocks pruningProcessor.updateBestFullBlock(header) log.info(s"Headers chain is likely synced after header ${header.encodedId} at height ${header.height}") diff --git a/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala b/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala index 288150c80e..02e2a64260 100644 --- a/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala +++ b/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala @@ -4,7 +4,6 @@ import org.ergoplatform.mining.difficulty.{DifficultyAdjustment, RequiredDifficu import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.settings.{Args, ErgoSettings, NetworkType} -import scorex.core.utils.NetworkTimeProvider import java.util.concurrent.TimeUnit import scala.collection.mutable @@ -16,11 +15,9 @@ object v2testing extends App { private val ergoSettings: ErgoSettings = ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/mainnet.conf"), Some(NetworkType.MainNet))) - val ntp = new NetworkTimeProvider(ergoSettings.scorexSettings.ntp) - val ldc = new DifficultyAdjustment(ergoSettings.chainSettings) - val eh = ErgoHistory.readOrGenerate(ergoSettings, ntp) + val eh = ErgoHistory.readOrGenerate(ergoSettings) val epochLength = ergoSettings.chainSettings.epochLength @@ -95,15 +92,13 @@ object AltDiff extends App { private val altSettings: ErgoSettings = ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/alt.conf"), Some(NetworkType.MainNet))) - val ntp = new NetworkTimeProvider(altSettings.scorexSettings.ntp) - val epochLength = altSettings.chainSettings.epochLength println(currentSettings.chainSettings.epochLength) println(altSettings.chainSettings.epochLength) - val eh = ErgoHistory.readOrGenerate(altSettings, ntp) + val eh = ErgoHistory.readOrGenerate(altSettings) println("best: " + eh.bestHeaderOpt.map(_.height)) @@ -130,7 +125,6 @@ object AdaptiveSimulator extends App { private val altSettings: ErgoSettings = ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/alt.conf"), Some(NetworkType.MainNet))) - val ntp = new NetworkTimeProvider(altSettings.scorexSettings.ntp) val ldc = new DifficultyAdjustment(altSettings.chainSettings) diff --git a/src/main/scala/scorex/core/app/Application.scala b/src/main/scala/scorex/core/app/Application.scala index a7224b5791..7e47ecddd0 100644 --- a/src/main/scala/scorex/core/app/Application.scala +++ b/src/main/scala/scorex/core/app/Application.scala @@ -17,7 +17,6 @@ import scorex.core.network.message.Message.MessageCode import scorex.core.network.message._ import scorex.core.network.peer.PeerManagerRef import scorex.core.settings.ScorexSettings -import scorex.core.utils.NetworkTimeProvider import scorex.util.ScorexLogging import java.net.InetSocketAddress @@ -68,8 +67,6 @@ trait Application extends ScorexLogging { /** API description in openapi format in YAML or JSON */ val swaggerConfig: String - val timeProvider = new NetworkTimeProvider(scorexSettings.ntp) - //an address to send to peers lazy val externalSocketAddress: Option[InetSocketAddress] = { scorexSettings.network.declaredAddress orElse { @@ -86,7 +83,6 @@ trait Application extends ScorexLogging { val scorexContext = ScorexContext( messageSpecs = basicSpecs ++ additionalMessageSpecs, upnpGateway = upnpGateway, - timeProvider = timeProvider, externalNodeAddress = externalSocketAddress ) diff --git a/src/main/scala/scorex/core/app/ScorexContext.scala b/src/main/scala/scorex/core/app/ScorexContext.scala index 82375253ff..6af361ab9a 100644 --- a/src/main/scala/scorex/core/app/ScorexContext.scala +++ b/src/main/scala/scorex/core/app/ScorexContext.scala @@ -4,9 +4,7 @@ import java.net.InetSocketAddress import scorex.core.network.UPnPGateway import scorex.core.network.message.MessageSpec -import scorex.core.utils.TimeProvider case class ScorexContext(messageSpecs: Seq[MessageSpec[_]], upnpGateway: Option[UPnPGateway], - timeProvider: TimeProvider, externalNodeAddress: Option[InetSocketAddress]) diff --git a/src/main/scala/scorex/core/network/NetworkController.scala b/src/main/scala/scorex/core/network/NetworkController.scala index 9ccdfd5c07..6890109704 100644 --- a/src/main/scala/scorex/core/network/NetworkController.scala +++ b/src/main/scala/scorex/core/network/NetworkController.scala @@ -9,13 +9,14 @@ import akka.util.Timeout import scorex.core.app.{ScorexContext, Version} import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.{DisconnectedPeer, HandshakedPeer} import org.ergoplatform.network.ModePeerFeature +import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.settings.ErgoSettings import scorex.core.network.message.Message.MessageCode import scorex.core.network.message.Message import scorex.core.network.peer.PeerManager.ReceivableMessages._ import scorex.core.network.peer.{LocalAddressPeerFeature, PeerInfo, PeerManager, PeersStatus, PenaltyType, RestApiUrlPeerFeature, SessionIdPeerFeature} -import scorex.core.utils.TimeProvider.Time -import scorex.core.utils.{NetworkUtils, TimeProvider} +import org.ergoplatform.nodeView.history.ErgoHistory.Time +import scorex.core.utils.NetworkUtils import scorex.util.ScorexLogging import scala.concurrent.ExecutionContext @@ -68,7 +69,7 @@ class NetworkController(ergoSettings: ErgoSettings, * Storing timestamp of a last message got via p2p network. * Used to check whether connectivity is lost. */ - private var lastIncomingMessageTime: TimeProvider.Time = 0L + private var lastIncomingMessageTime: ErgoHistory.Time = 0L //check own declared address for validity validateDeclaredAddress() @@ -111,7 +112,7 @@ class NetworkController(ergoSettings: ErgoSettings, context stop self } - private def networkTime(): Time = scorexContext.timeProvider.time() + private def time(): Time = System.currentTimeMillis() private def businessLogic: Receive = { //a message coming in from another peer @@ -125,7 +126,7 @@ class NetworkController(ergoSettings: ErgoSettings, val remoteAddress = remote.connectionId.remoteAddress connections.get(remoteAddress) match { case Some(cp) => - val now = networkTime() + val now = time() lastIncomingMessageTime = now cp.lastMessage = now case None => log.warn("Connection not found for a message got from: " + remoteAddress) @@ -180,7 +181,7 @@ class NetworkController(ergoSettings: ErgoSettings, handlerRef ! Close case Handshaked(connectedPeer) => - val now = networkTime() + val now = time() lastIncomingMessageTime = now handleHandshake(connectedPeer, sender()) @@ -194,7 +195,7 @@ class NetworkController(ergoSettings: ErgoSettings, // If a message received from p2p within connection timeout, // connectivity is not lost thus we're removing the peer // we add multiplier 6 to remove more dead peers (and still not dropping a lot when connectivity lost) - val noNetworkMessagesFor = networkTime() - lastIncomingMessageTime + val noNetworkMessagesFor = time() - lastIncomingMessageTime if (noNetworkMessagesFor < networkSettings.connectionTimeout.toMillis * 6) { peerManagerRef ! RemovePeer(c.remoteAddress) } @@ -218,7 +219,7 @@ class NetworkController(ergoSettings: ErgoSettings, //calls from API / application private def interfaceCalls: Receive = { case GetPeersStatus => - sender() ! PeersStatus(lastIncomingMessageTime, networkTime()) + sender() ! PeersStatus(lastIncomingMessageTime, time()) case GetConnectedPeers => sender() ! connections.values.filter(_.peerInfo.nonEmpty) @@ -286,7 +287,7 @@ class NetworkController(ergoSettings: ErgoSettings, context.system.scheduler.scheduleWithFixedDelay(60.seconds, 60.seconds) { () => { // Drop connections with peers if they seem to be inactive - val now = networkTime() + val now = time() connections.values.foreach { cp => val lastSeen = cp.lastMessage val timeout = networkSettings.inactiveConnectionDeadline.toMillis @@ -373,7 +374,7 @@ class NetworkController(ergoSettings: ErgoSettings, val handler = context.actorOf(handlerProps) // launch connection handler context.watch(handler) - val connectedPeer = ConnectedPeer(connectionId, handler, networkTime(), None) + val connectedPeer = ConnectedPeer(connectionId, handler, time(), None) connections += connectionId.remoteAddress -> connectedPeer unconfirmedConnections -= connectionId.remoteAddress } diff --git a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala index 6ef80138fb..ad3f850a9e 100644 --- a/src/main/scala/scorex/core/network/PeerConnectionHandler.scala +++ b/src/main/scala/scorex/core/network/PeerConnectionHandler.scala @@ -72,7 +72,7 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, val peerInfo = PeerInfo( receivedHandshake.peerSpec, - scorexContext.timeProvider.time(), + System.currentTimeMillis(), Some(direction) ) val peer = ConnectedPeer(connectionDescription.connectionId, self, 0, Some(peerInfo)) @@ -243,7 +243,7 @@ class PeerConnectionHandler(scorexSettings: ScorexSettings, ownSocketAddress, localFeatures ), - scorexContext.timeProvider.time() + System.currentTimeMillis() ) } diff --git a/src/main/scala/scorex/core/network/peer/PeerDatabase.scala b/src/main/scala/scorex/core/network/peer/PeerDatabase.scala index 841b9790fc..086ccba786 100644 --- a/src/main/scala/scorex/core/network/peer/PeerDatabase.scala +++ b/src/main/scala/scorex/core/network/peer/PeerDatabase.scala @@ -1,9 +1,10 @@ package scorex.core.network.peer +import org.ergoplatform.nodeView.history.ErgoHistory + import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream} import java.net.{InetAddress, InetSocketAddress} import org.ergoplatform.settings.ErgoSettings -import scorex.core.utils.TimeProvider import scorex.db.LDBFactory import scorex.util.ScorexLogging @@ -13,7 +14,7 @@ import scala.util.{Failure, Success, Try} /** * In-memory peer database implementation supporting temporal blacklisting. */ -final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) extends ScorexLogging { +final class PeerDatabase(settings: ErgoSettings) extends ScorexLogging { private val objectStore = LDBFactory.createKvDb(s"${settings.directory}/peers") @@ -29,7 +30,7 @@ final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) ext /** * banned peer ip -> ban expiration timestamp */ - private var blacklist = Map.empty[InetAddress, TimeProvider.Time] + private var blacklist = Map.empty[InetAddress, ErgoHistory.Time] /** * penalized peer ip -> (accumulated penalty score, last penalty timestamp) @@ -85,7 +86,7 @@ final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) ext Option(socketAddress.getAddress).foreach { address => penaltyBook -= address if (!blacklist.keySet.contains(address)){ - blacklist += address -> (timeProvider.time() + penaltyDuration(penaltyType)) + blacklist += address -> (System.currentTimeMillis() + penaltyDuration(penaltyType)) } else { log.warn(s"${address.toString} is already blacklisted") } @@ -126,7 +127,7 @@ final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) ext */ def penalize(socketAddress: InetSocketAddress, penaltyType: PenaltyType): Boolean = Option(socketAddress.getAddress).exists { address => - val currentTime = timeProvider.time() + val currentTime = System.currentTimeMillis() val safeInterval = settings.scorexSettings.network.penaltySafeInterval.toMillis val (penaltyScoreAcc, lastPenaltyTs) = penaltyBook.getOrElse(address, (0, 0L)) val applyPenalty = currentTime - lastPenaltyTs - safeInterval > 0 || penaltyType.isPermanent @@ -139,7 +140,7 @@ final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) ext if (newPenaltyScore > settings.scorexSettings.network.penaltyScoreThreshold) { true } else { - penaltyBook += address -> (newPenaltyScore -> timeProvider.time()) + penaltyBook += address -> (newPenaltyScore -> System.currentTimeMillis()) false } } @@ -154,7 +155,7 @@ final class PeerDatabase(settings: ErgoSettings, timeProvider: TimeProvider) ext Option(socketAddress.getAddress).map(penaltyScore).getOrElse(0) private def checkBanned(address: InetAddress, bannedTill: Long): Boolean = { - val stillBanned = timeProvider.time() < bannedTill + val stillBanned = System.currentTimeMillis() < bannedTill if (!stillBanned) removeFromBlacklist(address) stillBanned } diff --git a/src/main/scala/scorex/core/network/peer/PeerManager.scala b/src/main/scala/scorex/core/network/peer/PeerManager.scala index 94909682d3..82cc399493 100644 --- a/src/main/scala/scorex/core/network/peer/PeerManager.scala +++ b/src/main/scala/scorex/core/network/peer/PeerManager.scala @@ -19,7 +19,7 @@ class PeerManager(settings: ErgoSettings, scorexContext: ScorexContext) extends import PeerManager.ReceivableMessages._ - private val peerDatabase = new PeerDatabase(settings, scorexContext.timeProvider) + private val peerDatabase = new PeerDatabase(settings) if (peerDatabase.isEmpty) { // fill database with peers from config file if empty diff --git a/src/main/scala/scorex/core/settings/Settings.scala b/src/main/scala/scorex/core/settings/Settings.scala index ab96e5f675..c67890f61b 100644 --- a/src/main/scala/scorex/core/settings/Settings.scala +++ b/src/main/scala/scorex/core/settings/Settings.scala @@ -6,7 +6,6 @@ import com.typesafe.config.{Config, ConfigFactory} import net.ceedubs.ficus.Ficus._ import net.ceedubs.ficus.readers.ArbitraryTypeReader._ import scorex.core.network.message.Message -import scorex.core.utils.NetworkTimeProviderSettings import scorex.util.ScorexLogging import scala.concurrent.duration._ @@ -57,8 +56,7 @@ case class ScorexSettings(dataDir: File, logDir: File, logging: LoggingSettings, network: NetworkSettings, - restApi: RESTApiSettings, - ntp: NetworkTimeProviderSettings) + restApi: RESTApiSettings) object ScorexSettings extends ScorexLogging with SettingsReaders { diff --git a/src/main/scala/scorex/core/utils/NetworkTime.scala b/src/main/scala/scorex/core/utils/NetworkTime.scala deleted file mode 100644 index 528aa38b05..0000000000 --- a/src/main/scala/scorex/core/utils/NetworkTime.scala +++ /dev/null @@ -1,59 +0,0 @@ -package scorex.core.utils - -import java.net.InetAddress -import java.util.concurrent.atomic.AtomicLong - -import org.apache.commons.net.ntp.NTPUDPClient - -import scala.concurrent.duration._ -import scala.concurrent.{ExecutionContext, Future} -import scala.util.{Failure, Success} - -object NetworkTime { - def localWithOffset(offset: Long): Long = System.currentTimeMillis() + offset - type Offset = Long -} - -case class NetworkTimeProviderSettings(server: String, updateEvery: FiniteDuration, timeout: FiniteDuration) - -class NetworkTimeProvider(ntpSettings: NetworkTimeProviderSettings)(implicit ec: ExecutionContext) - extends TimeProvider with scorex.util.ScorexLogging { - - private val lastUpdate = new AtomicLong(0) - private val offset = new AtomicLong(0) - private val client = new NTPUDPClient() - client.setDefaultTimeout(ntpSettings.timeout.toMillis.toInt) - client.open() - - override def time(): TimeProvider.Time = { - checkUpdateRequired() - NetworkTime.localWithOffset(offset.get()) - } - - private def updateOffset(): Future[NetworkTime.Offset] = Future { - val info = client.getTime(InetAddress.getByName(ntpSettings.server)) - info.computeDetails() - info.getOffset - } - - private def checkUpdateRequired(): Unit = { - val time = NetworkTime.localWithOffset(offset.get()) - // set lastUpdate to current time so other threads won't start to update it - val lu = lastUpdate.getAndSet(time) - if (time > lu + ntpSettings.updateEvery.toMillis) { - // time to update offset - updateOffset().onComplete { - case Success(newOffset) => - offset.set(newOffset) - log.info("New offset adjusted: " + offset) - lastUpdate.set(time) - case Failure(e) => - log.warn("Problems with NTP: ", e) - lastUpdate.compareAndSet(time, lu) - } - } else { - // No update required. Set lastUpdate back to it's initial value - lastUpdate.compareAndSet(time, lu) - } - } -} diff --git a/src/main/scala/scorex/core/utils/TimeProvider.scala b/src/main/scala/scorex/core/utils/TimeProvider.scala deleted file mode 100644 index a2ab1e9b99..0000000000 --- a/src/main/scala/scorex/core/utils/TimeProvider.scala +++ /dev/null @@ -1,9 +0,0 @@ -package scorex.core.utils - -object TimeProvider { - type Time = Long -} - -trait TimeProvider { - def time(): TimeProvider.Time -} diff --git a/src/test/scala/org/ergoplatform/http/routes/ErgoPeersApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/ErgoPeersApiRouteSpec.scala index d28a8339a0..de1a6edc5d 100644 --- a/src/test/scala/org/ergoplatform/http/routes/ErgoPeersApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/ErgoPeersApiRouteSpec.scala @@ -16,8 +16,6 @@ import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks import scorex.core.network.NetworkController.ReceivableMessages.GetConnectedPeers import scorex.core.network.peer.PeerManager.ReceivableMessages.GetAllPeers import scorex.core.settings.RESTApiSettings -import scorex.core.utils.NetworkTimeProvider -import scorex.core.utils.TimeProvider.Time import java.net.InetSocketAddress import scala.concurrent.Future @@ -30,10 +28,6 @@ class ErgoPeersApiRouteSpec extends AnyFlatSpec with ScalaCheckPropertyChecks with Stubs { - val fakeTimeProvider: NetworkTimeProvider = new NetworkTimeProvider(settings.scorexSettings.ntp) { - override def time(): Time = 123 - } - implicit val actorTimeout: Timeout = Timeout(15.seconds.dilated) implicit val routeTimeout: RouteTestTimeout = RouteTestTimeout(15.seconds.dilated) diff --git a/src/test/scala/org/ergoplatform/http/routes/InfoApiRoutesSpec.scala b/src/test/scala/org/ergoplatform/http/routes/InfoApiRoutesSpec.scala index a599cb4ca6..ebe6f17110 100644 --- a/src/test/scala/org/ergoplatform/http/routes/InfoApiRoutesSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/InfoApiRoutesSpec.scala @@ -21,8 +21,6 @@ import org.ergoplatform.nodeView.history.ErgoHistory.Difficulty import org.ergoplatform.utils.Stubs import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import scorex.core.utils.TimeProvider.Time -import scorex.core.utils.NetworkTimeProvider import scala.concurrent.duration._ import scala.concurrent.{Await, Future} @@ -33,14 +31,10 @@ class InfoApiRoutesSpec extends AnyFlatSpec with FailFastCirceSupport with Stubs { - val fakeTimeProvider: NetworkTimeProvider = new NetworkTimeProvider(settings.scorexSettings.ntp) { - override def time(): Time = 123 - } - implicit val actorTimeout: Timeout = Timeout(15.seconds.dilated) implicit val routeTimeout: RouteTestTimeout = RouteTestTimeout(15.seconds.dilated) - val statsCollector: ActorRef = ErgoStatsCollectorRef(nodeViewRef, networkControllerRef, null, settings, fakeTimeProvider) - val route: Route = InfoApiRoute(statsCollector, settings.scorexSettings.restApi, fakeTimeProvider).route + val statsCollector: ActorRef = ErgoStatsCollectorRef(nodeViewRef, networkControllerRef, null, settings) + val route: Route = InfoApiRoute(statsCollector, settings.scorexSettings.restApi).route val requiredDifficulty = BigInt(1) override def beforeAll: Unit = { @@ -57,7 +51,7 @@ class InfoApiRoutesSpec extends AnyFlatSpec c.downField("appVersion").as[String] shouldEqual Right(Version.VersionString) c.downField("stateType").as[String] shouldEqual Right(settings.nodeSettings.stateType.stateTypeName) c.downField("isMining").as[Boolean] shouldEqual Right(settings.nodeSettings.mining) - c.downField("launchTime").as[Long] shouldEqual Right(fakeTimeProvider.time()) + (System.currentTimeMillis() - c.downField("launchTime").as[Long].toOption.getOrElse(0L)) < 2000 shouldBe true c.downField("eip27Supported").as[Boolean] shouldEqual Right(true) c.downField("restApiUrl").as[String] shouldEqual Right("https://example.com:80") } diff --git a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala index c65d89c64c..0bc8866630 100644 --- a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala @@ -52,7 +52,7 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event val testProbe = new TestProbe(system) system.eventStream.subscribe(testProbe.ref, newBlockSignal) - val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings, timeProvider) + val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(viewHolderRef) val candidateGenerator: ActorRef = @@ -60,7 +60,6 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event defaultMinerSecret.publicImage, readersHolderRef, viewHolderRef, - timeProvider, defaultSettings ) ErgoMiningThread(defaultSettings, candidateGenerator, defaultMinerSecret.w) @@ -75,7 +74,7 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event val testProbe = new TestProbe(system) system.eventStream.subscribe(testProbe.ref, newBlockSignal) - val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings, timeProvider) + val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(viewHolderRef) val candidateGenerator: ActorRef = @@ -83,7 +82,6 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event defaultMinerSecret.publicImage, readersHolderRef, viewHolderRef, - timeProvider, defaultSettings ) @@ -119,7 +117,7 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event val testProbe = new TestProbe(system) system.eventStream.subscribe(testProbe.ref, newBlockSignal) - val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings, timeProvider) + val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(viewHolderRef) val candidateGenerator: ActorRef = @@ -127,7 +125,6 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event defaultMinerSecret.publicImage, readersHolderRef, viewHolderRef, - timeProvider, defaultSettings ) @@ -159,7 +156,7 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event ) { val testProbe = new TestProbe(system) system.eventStream.subscribe(testProbe.ref, newBlockSignal) - val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings, timeProvider) + val viewHolderRef: ActorRef = ErgoNodeViewRef(defaultSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(viewHolderRef) val candidateGenerator: ActorRef = @@ -167,7 +164,6 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event defaultMinerSecret.publicImage, readersHolderRef, viewHolderRef, - timeProvider, defaultSettings ) diff --git a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala index 2dd69676ac..d17b3da442 100644 --- a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala @@ -68,14 +68,13 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen } complexScript.complexity shouldBe 28077 - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) val minerRef: ActorRef = ErgoMiner( ergoSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) expectNoMessage(1 second) @@ -145,7 +144,7 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen val testProbe = new TestProbe(system) system.eventStream.subscribe(testProbe.ref, newBlockSignal) - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) expectNoMessage(1 second) val r: Readers = requestReaders @@ -155,7 +154,6 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen ergoSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) minerRef ! StartMining @@ -220,14 +218,13 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen system.eventStream.subscribe(testProbe.ref, newBlockSignal) val ergoSettings: ErgoSettings = defaultSettings.copy(directory = createTempDir.getAbsolutePath) - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) val minerRef: ActorRef = ErgoMiner( ergoSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) expectNoMessage(1 second) @@ -296,14 +293,13 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen it should "prepare external candidate" in new TestKit(ActorSystem()) { val ergoSettings: ErgoSettings = defaultSettings.copy(directory = createTempDir.getAbsolutePath) - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) def minerRef: ActorRef = ErgoMiner( ergoSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) @@ -320,14 +316,13 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen system.eventStream.subscribe(testProbe.ref, newBlockSignal) val ergoSettings: ErgoSettings = defaultSettings.copy(directory = createTempDir.getAbsolutePath) - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) val minerRef: ActorRef = ErgoMiner( ergoSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) expectNoMessage(1 second) @@ -399,14 +394,13 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen empty.copy(nodeSettings = nodeSettings, chainSettings = chainSettings, directory = createTempDir.getAbsolutePath) } - val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(forkSettings, timeProvider) + val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(forkSettings) val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef) val minerRef: ActorRef = ErgoMiner( forkSettings, nodeViewHolderRef, readersHolderRef, - timeProvider, Some(defaultMinerSecret) ) diff --git a/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala b/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala index 5e8349e802..ba56bb1fed 100644 --- a/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala +++ b/src/test/scala/org/ergoplatform/network/ErgoNodeViewSynchronizerSpecification.scala @@ -24,7 +24,6 @@ import scorex.core.network.message._ import scorex.core.network.peer.PeerInfo import scorex.core.network.{ConnectedPeer, DeliveryTracker} import scorex.core.serialization.ScorexSerializer -import scorex.core.utils.NetworkTimeProvider import scorex.testkit.utils.AkkaFixture import scala.concurrent.duration.{Duration, _} @@ -54,13 +53,12 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc } } - class NodeViewHolderMock extends ErgoNodeViewHolder[UtxoState](settings, timeProvider) + class NodeViewHolderMock extends ErgoNodeViewHolder[UtxoState](settings) class SynchronizerMock(networkControllerRef: ActorRef, viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, - timeProvider: NetworkTimeProvider, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker) (implicit ec: ExecutionContext) extends ErgoNodeViewSynchronizer( @@ -68,15 +66,14 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc viewHolderRef, syncInfoSpec, settings, - timeProvider, syncTracker, deliveryTracker)(ec) { protected def broadcastInvForNewModifier(mod: PersistentNodeViewModifier): Unit = { mod match { - case fb: ErgoFullBlock if fb.header.isNew(timeProvider, 1.hour) => + case fb: ErgoFullBlock if fb.header.isNew(1.hour) => fb.toSeq.foreach(s => broadcastModifierInv(s)) - case h: Header if h.isNew(timeProvider, 1.hour) => + case h: Header if h.isNew(1.hour) => broadcastModifierInv(h) case _ => } @@ -132,11 +129,10 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc val settings = ErgoSettings.read() val pool = ErgoMemPool.empty(settings) implicit val ec: ExecutionContextExecutor = system.dispatcher - val tp = new NetworkTimeProvider(settings.scorexSettings.ntp) val ncProbe = TestProbe("NetworkControllerProbe") val pchProbe = TestProbe("PeerHandlerProbe") val eventListener = TestProbe("EventListener") - val syncTracker = ErgoSyncTracker(settings.scorexSettings.network, timeProvider) + val syncTracker = ErgoSyncTracker(settings.scorexSettings.network) val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(settings) // each test should always start with empty history @@ -149,7 +145,6 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc nodeViewHolderMockRef, ErgoSyncInfoMessageSpec, settings, - tp, syncTracker, deliveryTracker) )) @@ -157,7 +152,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc @SuppressWarnings(Array("org.wartremover.warts.OptionPartial")) val tx = validErgoTransactionGenTemplate(0, 0).sample.get._2 - val peerInfo = PeerInfo(defaultPeerSpec, timeProvider.time()) + val peerInfo = PeerInfo(defaultPeerSpec, System.currentTimeMillis()) @SuppressWarnings(Array("org.wartremover.warts.OptionPartial")) val p: ConnectedPeer = ConnectedPeer( connectionIdGen.sample.get, @@ -180,7 +175,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc implicit val ec: ExecutionContextExecutor = system.dispatcher val ncProbe = TestProbe("NetworkControllerProbe") val pchProbe = TestProbe("PeerHandlerProbe") - val syncTracker = ErgoSyncTracker(settings.scorexSettings.network, timeProvider) + val syncTracker = ErgoSyncTracker(settings.scorexSettings.network) val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(settings) // each test should always start with empty history @@ -193,12 +188,11 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc nodeViewHolderMockRef, ErgoSyncInfoMessageSpec, settings, - timeProvider, syncTracker, deliveryTracker) )) - val peerInfo = PeerInfo(defaultPeerSpec, timeProvider.time()) + val peerInfo = PeerInfo(defaultPeerSpec, System.currentTimeMillis()) @SuppressWarnings(Array("org.wartremover.warts.OptionPartial")) val peer: ConnectedPeer = ConnectedPeer( connectionIdGen.sample.get, @@ -254,7 +248,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc implicit val patienceConfig: PatienceConfig = PatienceConfig(5.second, 100.millis) // we generate and apply existing base chain - val hhistory = ErgoHistory.readOrGenerate(settings, timeProvider) + val hhistory = ErgoHistory.readOrGenerate(settings) val baseChain = genHeaderChain(_.size > 4, None, hhistory.difficultyCalculator, None, false) baseChain.headers.foreach(hhistory.append) val bestHeaderOpt = hhistory.bestHeaderOpt @@ -290,7 +284,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc } eventually { // test whether applied header was actually persisted to history - val hist = ErgoHistory.readOrGenerate(settings, timeProvider) + val hist = ErgoHistory.readOrGenerate(settings) hist.bestHeaderIdOpt.get shouldBe appliedHeader.id } } @@ -313,7 +307,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc // we generate fork of two headers, starting from the parent of the best header // so the depth of the rollback is 1, and the fork bypasses the best chain by 1 header - val hhistory = ErgoHistory.readOrGenerate(settings, timeProvider) + val hhistory = ErgoHistory.readOrGenerate(settings) val newHeaders = genHeaderChain(2, hhistory, diffBitsOpt = None, useRealTs = false).headers val newHistory = newHeaders.foldLeft(hhistory) { case (hist, header) => hist.append(header).get._1 } val parentOpt = newHistory.lastHeaders(2).headOption @@ -354,7 +348,7 @@ class ErgoNodeViewSynchronizerSpecification extends HistoryTestHelpers with Matc deliveryTracker.reset() - val hist = ErgoHistory.readOrGenerate(settings, timeProvider) + val hist = ErgoHistory.readOrGenerate(settings) // generate smaller fork that is going to be reverted after applying a bigger fork val smallFork = genChain(4, hist) diff --git a/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala b/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala index 02872c3449..e0d118fcdc 100644 --- a/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala +++ b/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala @@ -11,7 +11,7 @@ class ErgoSyncTrackerSpecification extends ErgoPropertyTest { val peerInfo = PeerInfo(defaultPeerSpec, time, Some(Incoming)) val cid = ConnectionId(inetAddr1, inetAddr2, Incoming) val connectedPeer = ConnectedPeer(cid, handlerRef = null, lastMessage = 5L, Some(peerInfo)) - val syncTracker = ErgoSyncTracker(settings.scorexSettings.network, timeProvider) + val syncTracker = ErgoSyncTracker(settings.scorexSettings.network) val height = 1000 // add peer to sync diff --git a/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala b/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala index e9f8471729..2a49a97174 100644 --- a/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala @@ -30,12 +30,12 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi // too big chain update delay val notAcceptableDelay = System.currentTimeMillis() - (initSettings.nodeSettings.acceptableChainUpdateDelay.toMillis + 100) val invalidProgress = ChainProgress(block, 2, 3, notAcceptableDelay) - ErgoNodeViewHolder.checkChainIsHealthy(invalidProgress, history, timeProvider, initSettings).isInstanceOf[ChainIsStuck] shouldBe true + ErgoNodeViewHolder.checkChainIsHealthy(invalidProgress, history, initSettings).isInstanceOf[ChainIsStuck] shouldBe true // acceptable chain update delay val acceptableDelay = System.currentTimeMillis() - 5 val validProgress = ChainProgress(block, 2, 3, acceptableDelay) - ErgoNodeViewHolder.checkChainIsHealthy(validProgress, history, timeProvider, initSettings) shouldBe ChainIsHealthy + ErgoNodeViewHolder.checkChainIsHealthy(validProgress, history, initSettings) shouldBe ChainIsHealthy } diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala index 0020660acf..e4d9de0836 100644 --- a/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala +++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala @@ -16,7 +16,6 @@ import org.ergoplatform.settings.Constants.HashLength import org.ergoplatform.utils.{ErgoTestHelpers, HistoryTestHelpers} import org.scalacheck.Gen import scorex.core.network.DeliveryTracker -import scorex.core.utils.NetworkTimeProvider import scorex.core.{PersistentNodeViewModifier, bytesToId} import scorex.crypto.authds.ADDigest import scorex.crypto.hash.{Blake2b256, Digest32} @@ -47,7 +46,7 @@ trait ErgoSanity[ST <: ErgoState[ST]] extends HistoryTests override lazy val memPoolGenerator: Gen[MPool] = emptyMemPoolGen override def syntacticallyValidModifier(history: HT): Header = { - val bestTimestamp = history.bestHeaderOpt.map(_.timestamp + 1).getOrElse(timeProvider.time()) + val bestTimestamp = history.bestHeaderOpt.map(_.timestamp + 1).getOrElse(System.currentTimeMillis()) powScheme.prove( history.bestHeaderOpt, @@ -56,7 +55,7 @@ trait ErgoSanity[ST <: ErgoState[ST]] extends HistoryTests ADDigest @@ Array.fill(HashLength + 1)(0.toByte), Digest32 @@ Array.fill(HashLength)(0.toByte), Digest32 @@ Array.fill(HashLength)(0.toByte), - Math.max(timeProvider.time(), bestTimestamp), + Math.max(System.currentTimeMillis(), bestTimestamp), Digest32 @@ Array.fill(HashLength)(0.toByte), Array.fill(3)(0: Byte), defaultMinerSecretNumber @@ -92,7 +91,6 @@ trait ErgoSanity[ST <: ErgoState[ST]] extends HistoryTests viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, - timeProvider: NetworkTimeProvider, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker) (implicit ec: ExecutionContext) extends ErgoNodeViewSynchronizer( @@ -100,15 +98,14 @@ trait ErgoSanity[ST <: ErgoState[ST]] extends HistoryTests viewHolderRef, syncInfoSpec, settings, - timeProvider, syncTracker, deliveryTracker)(ec) { protected def broadcastInvForNewModifier(mod: PersistentNodeViewModifier): Unit = { mod match { - case fb: ErgoFullBlock if fb.header.isNew(timeProvider, 1.hour) => + case fb: ErgoFullBlock if fb.header.isNew(1.hour) => fb.toSeq.foreach(s => broadcastModifierInv(s)) - case h: Header if h.isNew(timeProvider, 1.hour) => + case h: Header if h.isNew(1.hour) => broadcastModifierInv(h) case _ => } diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala index ef617beb32..db3bc448dd 100644 --- a/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala +++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala @@ -18,7 +18,6 @@ import scorex.core.idToBytes import scorex.core.network.{ConnectedPeer, DeliveryTracker} import scorex.core.network.peer.PeerInfo import scorex.core.serialization.ScorexSerializer -import scorex.core.utils.NetworkTimeProvider import scala.concurrent.ExecutionContextExecutor @@ -64,12 +63,11 @@ class ErgoSanityDigest extends ErgoSanity[DIGEST_ST] { val v = h.bestFullBlockIdOpt.orElse(h.bestHeaderIdOpt).get s.store.update(idToBytes(v), Seq(), Seq()).get implicit val ec: ExecutionContextExecutor = system.dispatcher - val tp = new NetworkTimeProvider(settings.scorexSettings.ntp) val ncProbe = TestProbe("NetworkControllerProbe") val vhProbe = TestProbe("ViewHolderProbe") val pchProbe = TestProbe("PeerHandlerProbe") val eventListener = TestProbe("EventListener") - val syncTracker = ErgoSyncTracker(settings.scorexSettings.network, timeProvider) + val syncTracker = ErgoSyncTracker(settings.scorexSettings.network) val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(settings) val ref = system.actorOf(Props( new SyncronizerMock( @@ -77,7 +75,6 @@ class ErgoSanityDigest extends ErgoSanity[DIGEST_ST] { vhProbe.ref, ErgoSyncInfoMessageSpec, settings, - tp, syncTracker, deliveryTracker ) diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala index 558d2bec8b..25c5862f33 100644 --- a/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala +++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanityUTXO.scala @@ -18,7 +18,6 @@ import org.scalacheck.Gen import scorex.core.network.{ConnectedPeer, DeliveryTracker} import scorex.core.network.peer.PeerInfo import scorex.core.serialization.ScorexSerializer -import scorex.core.utils.NetworkTimeProvider import scala.concurrent.ExecutionContextExecutor @@ -59,12 +58,11 @@ class ErgoSanityUTXO extends ErgoSanity[UTXO_ST] with ErgoTestHelpers { val settings = ErgoSettings.read() val pool = ErgoMemPool.empty(settings) implicit val ec: ExecutionContextExecutor = system.dispatcher - val tp = new NetworkTimeProvider(settings.scorexSettings.ntp) val ncProbe = TestProbe("NetworkControllerProbe") val vhProbe = TestProbe("ViewHolderProbe") val pchProbe = TestProbe("PeerHandlerProbe") val eventListener = TestProbe("EventListener") - val syncTracker = ErgoSyncTracker(settings.scorexSettings.network, timeProvider) + val syncTracker = ErgoSyncTracker(settings.scorexSettings.network) val deliveryTracker: DeliveryTracker = DeliveryTracker.empty(settings) val ref = system.actorOf(Props( new SyncronizerMock( @@ -72,7 +70,6 @@ class ErgoSanityUTXO extends ErgoSanity[UTXO_ST] with ErgoTestHelpers { vhProbe.ref, ErgoSyncInfoMessageSpec, settings, - tp, syncTracker, deliveryTracker) )) @@ -81,7 +78,7 @@ class ErgoSanityUTXO extends ErgoSanity[UTXO_ST] with ErgoTestHelpers { val tx = validErgoTransactionGenTemplate(minAssets = 0, maxAssets = 0).sample.get._2 - val peerInfo = PeerInfo(defaultPeerSpec, timeProvider.time()) + val peerInfo = PeerInfo(defaultPeerSpec, System.currentTimeMillis()) @SuppressWarnings(Array("org.wartremover.warts.OptionPartial")) val p: ConnectedPeer = ConnectedPeer( connectionIdGen.sample.get, diff --git a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala index 789a7b06fc..e3cb8e2112 100644 --- a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala +++ b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala @@ -51,7 +51,7 @@ object ChainGenerator extends App with ErgoTestHelpers { val boxSelector: BoxSelector = new ReplaceCompactCollectBoxSelector(30, 2, None) - val startTime = args.headOption.map(_.toLong).getOrElse(timeProvider.time - (blockInterval * 10).toMillis) + val startTime = args.headOption.map(_.toLong).getOrElse(System.currentTimeMillis() - (blockInterval * 10).toMillis) val dir = if (args.length < 2) new File("/tmp/ergo/data") else new File(args(1)) val txsSize: Int = if (args.length < 3) 100 * 1024 else args(2).toInt @@ -76,7 +76,7 @@ object ChainGenerator extends App with ErgoTestHelpers { val votingEpochLength = votingSettings.votingLength val protocolVersion = fullHistorySettings.chainSettings.protocolVersion - val history = ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + val history = ErgoHistory.readOrGenerate(fullHistorySettings) HistoryTestHelpers.allowToApplyOldBlocks(history) val (state, _) = ErgoState.generateGenesisUtxoState(stateDir, StateConstants(fullHistorySettings)) log.info(s"Going to generate a chain at ${dir.getAbsoluteFile} starting from ${history.bestFullBlockOpt}") @@ -93,7 +93,7 @@ object ChainGenerator extends App with ErgoTestHelpers { last: Option[Header], acc: Seq[ModifierId]): Seq[ModifierId] = { val time: Long = last.map(_.timestamp + blockInterval.toMillis).getOrElse(startTime) - if (time < timeProvider.time) { + if (time < System.currentTimeMillis()) { val (txs, lastOut) = genTransactions(last.map(_.height).getOrElse(ErgoHistory.GenesisHeight), initBox, state.stateContext) diff --git a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala index d3c02e37f2..7dabb08224 100644 --- a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala +++ b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala @@ -18,7 +18,6 @@ import org.ergoplatform.wallet.secrets.ExtendedSecretKey import org.ergoplatform.{DataInput, ErgoBox, ErgoScriptPredef} import scorex.core.app.Version import scorex.core.network.PeerSpec -import scorex.core.utils.NetworkTimeProvider import scorex.crypto.authds.ADDigest import scorex.crypto.hash.Digest32 import scorex.util.ScorexLogging @@ -47,7 +46,6 @@ trait ErgoTestConstants extends ScorexLogging { Parameters(0, Parameters.DefaultParameters ++ extension, ErgoValidationSettingsUpdate.empty) } - val timeProvider: NetworkTimeProvider = ErgoTestHelpers.defaultTimeProvider val initSettings: ErgoSettings = ErgoSettings.read(Args(Some("src/test/resources/application.conf"), None)) implicit val settings: ErgoSettings = initSettings diff --git a/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala b/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala index 9888f3583f..bf9d61ac3e 100644 --- a/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala +++ b/src/test/scala/org/ergoplatform/utils/ErgoTestHelpers.scala @@ -1,11 +1,10 @@ package org.ergoplatform.utils import org.ergoplatform.ErgoBoxCandidate -import org.ergoplatform.settings.ErgoSettings import org.ergoplatform.utils.generators.ValidBlocksGenerators import org.scalatest.{EitherValues, OptionValues} import scorex.core.network.peer.PeerInfo -import scorex.core.utils.{NetworkTimeProvider, ScorexEncoding} +import scorex.core.utils.ScorexEncoding import scorex.util.ScorexLogging import java.net.InetSocketAddress @@ -37,8 +36,8 @@ trait ErgoTestHelpers val inetAddr2 = new InetSocketAddress("93.93.93.93", 27017) val peers: Map[InetSocketAddress, PeerInfo] = Map( - inetAddr1 -> PeerInfo(defaultPeerSpec.copy(nodeName = "first"), timeProvider.time()), - inetAddr2 -> PeerInfo(defaultPeerSpec.copy(nodeName = "second"), timeProvider.time()) + inetAddr1 -> PeerInfo(defaultPeerSpec.copy(nodeName = "first"), System.currentTimeMillis()), + inetAddr2 -> PeerInfo(defaultPeerSpec.copy(nodeName = "second"), System.currentTimeMillis()) ) } @@ -46,7 +45,4 @@ object ErgoTestHelpers { implicit val defaultExecutionContext: ExecutionContext = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(10)) - - val defaultTimeProvider: NetworkTimeProvider = - new NetworkTimeProvider(ErgoSettings.read().scorexSettings.ntp) } diff --git a/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala b/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala index 628090c90a..256d1e56a3 100644 --- a/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala +++ b/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala @@ -65,7 +65,7 @@ trait HistoryTestHelpers extends ErgoPropertyTest { val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, chainSettings, nodeSettings, scorexSettings, walletSettings, settings.cacheSettings) - ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + ErgoHistory.readOrGenerate(fullHistorySettings) } } diff --git a/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala b/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala index 081a93dde3..116bfcf0b1 100644 --- a/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala +++ b/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala @@ -96,7 +96,7 @@ trait NodeViewBaseOps extends ErgoTestHelpers { def makeNextBlock(utxoState: UtxoState, txs: Seq[ErgoTransaction]) (implicit ctx: Ctx): ErgoFullBlock = { - val time = timeProvider.time() + val time = System.currentTimeMillis() val parent = getHistory.bestFullBlockOpt validFullBlock(parent, utxoState, txs, Some(time)) } diff --git a/src/test/scala/org/ergoplatform/utils/Stubs.scala b/src/test/scala/org/ergoplatform/utils/Stubs.scala index d1a1863bf6..1156e151d1 100644 --- a/src/test/scala/org/ergoplatform/utils/Stubs.scala +++ b/src/test/scala/org/ergoplatform/utils/Stubs.scala @@ -383,11 +383,11 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, chainSettings, nodeSettings, scorexSettings, walletSettings, settings.cacheSettings) - ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider) + ErgoHistory.readOrGenerate(fullHistorySettings) } def syntacticallyValidModifier(history: HT): Header = { - val bestTimestamp = history.bestHeaderOpt.map(_.timestamp + 1).getOrElse(timeProvider.time()) + val bestTimestamp = history.bestHeaderOpt.map(_.timestamp + 1).getOrElse(System.currentTimeMillis()) powScheme.prove( history.bestHeaderOpt, @@ -396,7 +396,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with ADDigest @@ Array.fill(HashLength + 1)(0.toByte), Digest32 @@ Array.fill(HashLength)(0.toByte), Digest32 @@ Array.fill(HashLength)(0.toByte), - Math.max(timeProvider.time(), bestTimestamp), + Math.max(System.currentTimeMillis(), bestTimestamp), Digest32 @@ Array.fill(HashLength)(0.toByte), Array.fill(3)(0: Byte), defaultMinerSecretNumber diff --git a/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala b/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala index 0249379336..5127620226 100644 --- a/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala +++ b/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala @@ -5,9 +5,8 @@ import akka.testkit.TestProbe import org.ergoplatform.mining.emission.EmissionRules import org.ergoplatform.nodeView.ErgoNodeViewRef import org.ergoplatform.settings.{ErgoSettings, Parameters} -import org.ergoplatform.utils.{ErgoTestHelpers, NodeViewTestContext} +import org.ergoplatform.utils.NodeViewTestContext import org.ergoplatform.wallet.utils.TestFileUtils -import scorex.core.utils.NetworkTimeProvider import scala.concurrent.ExecutionContext @@ -22,9 +21,8 @@ class NodeViewFixture(protoSettings: ErgoSettings, parameters: Parameters) exten val nodeViewDir: java.io.File = createTempDir @volatile var settings: ErgoSettings = protoSettings.copy(directory = nodeViewDir.getAbsolutePath) - val timeProvider: NetworkTimeProvider = ErgoTestHelpers.defaultTimeProvider val emission: EmissionRules = new EmissionRules(settings.chainSettings.monetary) - @volatile var nodeViewHolderRef: ActorRef = ErgoNodeViewRef(settings, timeProvider) + @volatile var nodeViewHolderRef: ActorRef = ErgoNodeViewRef(settings) val testProbe = new TestProbe(actorSystem) /** This sender should be imported to make TestProbe work! */ @@ -33,7 +31,7 @@ class NodeViewFixture(protoSettings: ErgoSettings, parameters: Parameters) exten def apply[T](test: self.type => T): T = try test(self) finally stop() def startNodeViewHolder(): Unit = { - nodeViewHolderRef = ErgoNodeViewRef(settings, timeProvider) + nodeViewHolderRef = ErgoNodeViewRef(settings) } def stopNodeViewHolder(): Unit = { diff --git a/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala b/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala index 08bd42a7ca..441bd039f2 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ChainGenerator.scala @@ -108,7 +108,7 @@ trait ChainGenerator extends ErgoTestConstants { EmptyDigest32, EmptyDigest32, tsOpt.getOrElse(prev.map(_.timestamp + control.desiredInterval.toMillis) - .getOrElse(if (useRealTs) timeProvider.time() else 0)), + .getOrElse(if (useRealTs) System.currentTimeMillis() else 0)), extensionHash, Array.fill(3)(0: Byte), defaultMinerSecretNumber @@ -163,7 +163,7 @@ trait ChainGenerator extends ErgoTestConstants { EmptyStateRoot, emptyProofs, txs, - Math.max(timeProvider.time(), prev.map(_.header.timestamp + 1).getOrElse(timeProvider.time())), + Math.max(System.currentTimeMillis(), prev.map(_.header.timestamp + 1).getOrElse(System.currentTimeMillis())), validExtension, Array.fill(3)(0: Byte), defaultMinerSecretNumber diff --git a/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala index a96af11484..31f60d02cb 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala @@ -202,7 +202,7 @@ trait ValidBlocksGenerators val (adProofBytes, updStateDigest) = utxoState.proofsForTransactions(transactions).get - val time = timeOpt.orElse(parentOpt.map(_.header.timestamp + 1)).getOrElse(timeProvider.time()) + val time = timeOpt.orElse(parentOpt.map(_.header.timestamp + 1)).getOrElse(System.currentTimeMillis()) val interlinks = parentOpt.toSeq.flatMap { block => nipopowAlgos.updateInterlinks(block.header, NipopowAlgos.unpackInterlinks(block.extension.fields).get) } @@ -231,7 +231,7 @@ trait ValidBlocksGenerators val (adProofBytes, updStateDigest) = wrappedState.proofsForTransactions(transactions).get - val time = timeOpt.orElse(parentOpt.map(_.timestamp + 1)).getOrElse(timeProvider.time()) + val time = timeOpt.orElse(parentOpt.map(_.timestamp + 1)).getOrElse(System.currentTimeMillis()) val interlinksExtension = nipopowAlgos.interlinksToExtension(nipopowAlgos.updateInterlinks(parentOpt, parentExtensionOpt)) val extension: ExtensionCandidate = parameters.toExtensionCandidate ++ interlinksExtension val votes = Array.fill(3)(0: Byte) From a91a6a11786bac1561a196d2942934b8859ceb50 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Sat, 28 Jan 2023 04:13:32 +0300 Subject: [PATCH 60/90] after-merging fixes --- .../nodeView/history/extra/ExtraIndexerSpecification.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala index 410859d1df..c51c72c9d1 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala @@ -47,7 +47,7 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, initSettings.chainSettings, nodeSettings, settings.scorexSettings, settings.walletSettings, settings.cacheSettings) - _history = ErgoHistory.readOrGenerate(fullHistorySettings, timeProvider)(null) + _history = ErgoHistory.readOrGenerate(fullHistorySettings)(null) ChainGenerator.generate(HEIGHT, dir)(_history) @@ -149,7 +149,7 @@ object ChainGenerator extends ErgoTestHelpers { stateDir.mkdirs() val (state, _) = ErgoState.generateGenesisUtxoState(stateDir, StateConstants(initSettings)) System.out.println(s"Going to generate a chain at ${dir.getAbsolutePath} starting from ${history.bestFullBlockOpt}") - startTime = timeProvider.time - (blockInterval * (length - 1)).toMillis + startTime = System.currentTimeMillis() - (blockInterval * (length - 1)).toMillis val chain = loop(state, None, None, Seq())(history) System.out.println(s"Chain of length ${chain.length} generated") history.bestHeaderOpt shouldBe history.bestFullBlockOpt.map(_.header) @@ -163,7 +163,7 @@ object ChainGenerator extends ErgoTestHelpers { last: Option[Header], acc: Seq[ModifierId])(history: ErgoHistory): Seq[ModifierId] = { val time: Long = last.map(_.timestamp + blockInterval.toMillis).getOrElse(startTime) - if (time < timeProvider.time) { + if (time < System.currentTimeMillis()) { val (txs, lastOut) = genTransactions(last.map(_.height).getOrElse(ErgoHistory.GenesisHeight), initBox, state.stateContext) From 7d22bba4067222197120f5fdbb4c4f7dea692bf5 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Mon, 30 Jan 2023 14:18:07 +0300 Subject: [PATCH 61/90] readme update --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 0f602c16ee..053f98c195 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,17 @@ heavy validation attacks A [White Paper](https://ergoplatform.org/docs/whitepaper.pdf) with a brief description is available. A Yellow Paper with detailed specification is underway and will be available shortly. At the moment, there are [drafts of the Yellow Paper](https://github.com/ergoplatform/ergo/tree/master/papers/yellow) available, and currently the reference implementation code should be considered as the specification. +## Security assumptions + +This client relies on some assumptions in regards with its environment: + +* execution environment is trusted. While seed is stored in encrypted files, and the client's + wallet tries to remove secret key from memory as soon as possible when it is not needed, the + client has no any protection from side-channel attacks, memory scans etc. +* clocks should be more or less synchronized. If timestamp of a block is more than 20 minutes + in future, the block will be temporarily rejected. The client does not use NTP or other time + syncing protocols. + ## Building and Running Node and UI See [documentation](https://docs.ergoplatform.com/node/install/) From f9f86b0458d837eb64148e70c7a8735da67b065f Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Mon, 30 Jan 2023 14:34:36 +0300 Subject: [PATCH 62/90] private acess for different methods and fields --- src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala | 3 +++ .../scala/org/ergoplatform/local/ErgoStatsCollector.scala | 2 +- .../org/ergoplatform/network/ErgoNodeViewSynchronizer.scala | 4 ++-- .../scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala | 4 ++-- .../history/storage/modifierprocessors/HeadersProcessor.scala | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala index ad7835f59b..ce04c7fbe8 100644 --- a/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/InfoApiRoute.scala @@ -9,6 +9,9 @@ import scorex.core.api.http.ApiResponse import scorex.core.settings.RESTApiSettings +/** + * API methods corresponding to /info path + */ case class InfoApiRoute(statsCollector: ActorRef, settings: RESTApiSettings) (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute { diff --git a/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala b/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala index 4e1d4fc392..8756d159aa 100644 --- a/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala +++ b/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala @@ -238,7 +238,7 @@ object ErgoStatsCollector { object ErgoStatsCollectorRef { - def props(readersHolder: ActorRef, + private def props(readersHolder: ActorRef, networkController: ActorRef, syncTracker : ErgoSyncTracker, settings: ErgoSettings): Props = diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 83722d78fd..4b091327ea 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -1015,7 +1015,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } } - protected def viewHolderEvents(historyReader: ErgoHistory, + private def viewHolderEvents(historyReader: ErgoHistory, mempoolReader: ErgoMemPool, blockAppliedTxsCache: FixedSizeApproximateCacheQueue): Receive = { // Requests BlockSections with `Unknown` status that are defined by block headers but not downloaded yet. @@ -1182,7 +1182,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, object ErgoNodeViewSynchronizer { - def props(networkControllerRef: ActorRef, + private def props(networkControllerRef: ActorRef, viewHolderRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index 5fd4346db8..153b397fe7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -658,7 +658,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti if (mempool) sender() ! ChangedMempool(nodeView._4.getReader) } - protected def handleHealthCheck: Receive = { + private def handleHealthCheck: Receive = { case IsChainHealthy => log.info(s"Check that chain is healthy, progress is $chainProgress") val healthCheckReply = chainProgress.map { progress => @@ -780,7 +780,7 @@ object ErgoNodeViewRef { private def utxoProps(settings: ErgoSettings): Props = Props.create(classOf[UtxoNodeViewHolder], settings) - def props(settings: ErgoSettings): Props = + private def props(settings: ErgoSettings): Props = (settings.nodeSettings.stateType match { case StateType.Digest => digestProps(settings) case StateType.Utxo => utxoProps(settings) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala index 994c7155ea..18d7f8f58d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala @@ -311,7 +311,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score private def validationState: ValidationState[Unit] = ModifierValidator(ErgoValidationSettings.initial) - private def time() = System.currentTimeMillis() + private def time(): ErgoHistory.Time = System.currentTimeMillis() def validate(header: Header): ValidationResult[Unit] = { if (header.isGenesis) { From cedc212035dd141c6ee22dbf3e9f25923218f7b8 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Mon, 30 Jan 2023 14:42:39 +0300 Subject: [PATCH 63/90] DifficultyEstimation.scala removed --- .../tools/DifficultyEstimation.scala | 230 ------------------ 1 file changed, 230 deletions(-) delete mode 100644 src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala diff --git a/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala b/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala deleted file mode 100644 index 9effea62ec..0000000000 --- a/src/main/scala/org/ergoplatform/tools/DifficultyEstimation.scala +++ /dev/null @@ -1,230 +0,0 @@ -package org.ergoplatform.tools - -import org.ergoplatform.mining.difficulty.{DifficultyAdjustment, RequiredDifficulty} -import org.ergoplatform.modifiers.history.header.Header -import org.ergoplatform.nodeView.history.ErgoHistory -import org.ergoplatform.settings.{Args, ErgoSettings, NetworkType} - -import java.util.concurrent.TimeUnit -import scala.collection.mutable -import scala.concurrent.duration.FiniteDuration -import scala.util.Random - -object v2testing extends App { - implicit val ec: scala.concurrent.ExecutionContext = scala.concurrent.ExecutionContext.global - - private val ergoSettings: ErgoSettings = ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/mainnet.conf"), Some(NetworkType.MainNet))) - - val ldc = new DifficultyAdjustment(ergoSettings.chainSettings) - - val eh = ErgoHistory.readOrGenerate(ergoSettings)(null) - - val epochLength = ergoSettings.chainSettings.epochLength - - println("Chain settings: " + ergoSettings.chainSettings) - println("Epoch length: " + epochLength) - println("Block time: " + ergoSettings.chainSettings.blockInterval) - - println("best: " + eh.bestHeaderOpt.map(_.height)) - - val heights = ldc.previousHeadersRequiredForRecalculation(843776 + 1 + 1024, epochLength) - - println("hs: " + heights) - - val headerOpts = heights.map(eh.bestHeaderIdAtHeight).map(idOpt => idOpt.flatMap(id => eh.typedModifierById[Header](id))) - - println(headerOpts.map(_.map(_.id))) - - println("dir: " + ergoSettings.directory) - - val h1 = headerOpts.head.get.copy(height = 843776 + 1024, nBits = 122447235L, timestamp = System.currentTimeMillis() + 1000*60*1440*6) - - val headers = headerOpts.flatten.toSeq ++ Seq(h1) - - val diff1 = ldc.calculate(headers, epochLength) - println("diff1: " + diff1) - val nbits1 = RequiredDifficulty.encodeCompactBits(diff1) - - val h2 = headerOpts.head.get.copy(height = 843776 + 2048, nBits = nbits1, timestamp = System.currentTimeMillis() + 1000*60*1440*10) - - val headers2 = headerOpts.flatten.toSeq.tail ++ Seq(h1, h2) - val diff2 = ldc.calculate(headers2, epochLength) - println("diff2: " + diff2) - val nbits2 = RequiredDifficulty.encodeCompactBits(diff2) - - val h3 = headerOpts.head.get.copy(height = 843776 + 3072, nBits = nbits2, timestamp = System.currentTimeMillis() + (1000*60*1440*11).toInt) - val headers3 = headers2.tail ++ Seq(h3) - val diff3 = ldc.calculate(headers3, epochLength) - println("diff3: " + diff3) - val nbits3 = RequiredDifficulty.encodeCompactBits(diff3) - - val h4 = headerOpts.head.get.copy(height = 843776 + 4096, nBits = nbits3, timestamp = System.currentTimeMillis() + (1000*60*1440*11.25).toInt) - val headers4 = headers3.tail ++ Seq(h4) - val diff4 = ldc.calculate(headers4, epochLength) - println("diff4: " + diff4) - val nbits4 = RequiredDifficulty.encodeCompactBits(diff4) - - - val h5 = headerOpts.head.get.copy(height = 843776 + 4096 + 1024, nBits = nbits3, timestamp = System.currentTimeMillis() + (1000*60*1440*13.25).toInt) - val headers5 = headers4.tail ++ Seq(h5) - val diff5 = ldc.calculate(headers5, epochLength) - println("diff5: " + diff5) - val nbits5 = RequiredDifficulty.encodeCompactBits(diff5) - - - val h6 = headerOpts.head.get.copy(height = 843776 + 4096 + 2048, nBits = nbits3, timestamp = System.currentTimeMillis() + (1000*60*1440*16.5).toInt) - val headers6 = headers5.tail ++ Seq(h6) - val diff6 = ldc.calculate(headers6, epochLength) - println("diff6: " + diff6) - val nbits6 = RequiredDifficulty.encodeCompactBits(diff6) - -} - - - -object AltDiff extends App { - - implicit val ec: scala.concurrent.ExecutionContext = scala.concurrent.ExecutionContext.global - - private val currentSettings: ErgoSettings = - ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/mainnet.conf"), Some(NetworkType.MainNet))) - - private val altSettings: ErgoSettings = - ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/alt.conf"), Some(NetworkType.MainNet))) - - val epochLength = altSettings.chainSettings.epochLength - - - println(currentSettings.chainSettings.epochLength) - println(altSettings.chainSettings.epochLength) - - val eh = ErgoHistory.readOrGenerate(altSettings)(null) - - println("best: " + eh.bestHeaderOpt.map(_.height)) - - val ldc = new DifficultyAdjustment(altSettings.chainSettings) - - (1 to 843207).foreach{h => - if(h % 1024 == 1 && h > 1) { - val heights = ldc.previousHeadersRequiredForRecalculation(h, epochLength) - val headers = heights.map(eh.bestHeaderIdAtHeight).map(idOpt => idOpt.flatMap(id => eh.typedModifierById[Header](id))).flatten - val calcDiff = ldc.eip37Calculate(headers, epochLength) - val chainDiff = eh.bestHeaderAtHeight(h).get.requiredDifficulty - - println(s"calculated diff for $h: $calcDiff, chain diff: $chainDiff , difference: ${calcDiff*100/chainDiff-100}%") - } - } - -} - - -object AdaptiveSimulator extends App { - import io.circe.parser._ - - implicit val ec: scala.concurrent.ExecutionContext = scala.concurrent.ExecutionContext.global - - private val altSettings: ErgoSettings = - ErgoSettings.read(Args(Some("/home/kushti/ergo/mainnet/alt.conf"), Some(NetworkType.MainNet))) - - val ldc = new DifficultyAdjustment(altSettings.chainSettings) - - val h1Json = - """ - |{ - | "extensionId" : "af4c9de8106960b47964d21e6eb2acdad7e3e168791e595f0806ebfb036ee7de", - | "difficulty" : "1199990374400", - | "votes" : "000000", - | "timestamp" : 1561978977137, - | "size" : 279, - | "stateRoot" : "18b7a08878f2a7ee4389c5a1cece1e2724abe8b8adc8916240dd1bcac069177303", - | "height" : 1, - | "nBits" : 100734821, - | "version" : 1, - | "id" : "b0244dfc267baca974a4caee06120321562784303a8a688976ae56170e4d175b", - | "adProofsRoot" : "766ab7a313cd2fb66d135b0be6662aa02dfa8e5b17342c05a04396268df0bfbb", - | "transactionsRoot" : "93fb06aa44413ff57ac878fda9377207d5db0e78833556b331b4d9727b3153ba", - | "extensionHash" : "0e5751c026e543b2e8ab2eb06099daa1d1e5df47778f7787faab45cdf12fe3a8", - | "powSolutions" : { - | "pk" : "03be7ad70c74f691345cbedba19f4844e7fc514e1188a7929f5ae261d5bb00bb66", - | "w" : "02da9385ac99014ddcffe88d2ac5f28ce817cd615f270a0a5eae58acfb9fd9f6a0", - | "n" : "000000030151dc63", - | "d" : 46909460813884299753486408728361968139945651324239558400157099627 - | }, - | "adProofsId" : "cfc4af9743534b30ef38deec118a85ce6f0a3741b79b7d294f3e089c118188dc", - | "transactionsId" : "fc13e7fd2d1ddbd10e373e232814b3c9ee1b6fbdc4e6257c288ecd9e6da92633", - | "parentId" : "0000000000000000000000000000000000000000000000000000000000000000" - |}""".stripMargin - - val h1 = Header.jsonDecoder.decodeJson(parse(h1Json).toOption.get).toOption.get - - var totalError = 0 - var maxDelay = 0 - - (1 to 100).foreach { _ => - var blockDelay = altSettings.chainSettings.blockInterval - val epochLength = altSettings.chainSettings.epochLength - var price = 100000 - - val medianChange = 3 // price going up 1% epoch on average - val variance = 10 // with +-20% - - val blocks = mutable.Map[Int, Header]() - blocks.put(0, h1.copy(height = 0)) // put genesis block - blocks.put(1, h1) // put genesis block - - val precision = BigInt("10000000000000000") - // t = d*c / p - // c = t*p/d - val c = blockDelay.toMillis * price * precision / h1.requiredDifficulty - println("c: " + c) - - - 129.to(32 * 1024 + 1, 128).foreach { h => - println("================================") - println("height: " + h) - - val newPrice = price + - Random.nextInt(price * medianChange / 100) + - (if (Random.nextBoolean()) { - Random.nextInt(price * variance / 100) - } else { - -Random.nextInt(price * variance / 100) - }) - /* - val epoch = (h - 1) / 128 - val newPrice = if(epoch%16 <8){ - price + Random.nextInt(price * variance / 100) - } else { - price - Random.nextInt(price * variance / 100) - } */ - - println("price: " + newPrice) - - val newBlockDelay = (blocks(h - 128).requiredDifficulty * c / precision / newPrice).toLong - - val blockBefore = h1.copy(height = h - 1, timestamp = blocks(h - 128).timestamp + 127 * newBlockDelay, nBits = blocks(h - 128).nBits) - blocks.put(h - 1, blockBefore) - - val heights = ldc.previousHeadersRequiredForRecalculation(h, epochLength) - val hs = heights.map(blocks.apply) - - val newDiff = ldc.eip37Calculate(hs, epochLength) - println("newDiff: " + newDiff) - val block = h1.copy(height = h, timestamp = blockBefore.timestamp + newBlockDelay, nBits = RequiredDifficulty.encodeCompactBits(newDiff)) - blocks.put(h, block) - - price = newPrice - blockDelay = FiniteDuration(newBlockDelay, TimeUnit.MILLISECONDS) - println("block delay: " + blockDelay.toSeconds + " s.") - totalError += Math.abs(altSettings.chainSettings.blockInterval.toMillis - blockDelay.toMillis).toInt - if (blockDelay.toSeconds > maxDelay) { - maxDelay = blockDelay.toSeconds.toInt - } - } - } - - println("Planned block time: " + altSettings.chainSettings.blockInterval) - println("Total error: " + totalError / 1000) - println("Max delay: " + maxDelay) - -} From 8a890f14da35ce40854175947a3039569b984d49 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Mon, 30 Jan 2023 15:22:51 +0300 Subject: [PATCH 64/90] scaladoc --- src/main/scala/org/ergoplatform/ErgoApp.scala | 4 ++++ .../ergoplatform/network/ErgoSyncTracker.scala | 16 ++++++++++++++-- .../nodeView/history/ErgoHistory.scala | 8 ++++++++ .../modifierprocessors/HeadersProcessor.scala | 4 ++-- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index c25e159379..b0f3e3ebc0 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -28,6 +28,10 @@ import java.net.InetSocketAddress import scala.concurrent.{ExecutionContext, Future} import scala.io.{Codec, Source} +/** + * Ergo reference protocol client application runnable from command line + * @param args parsed command line arguments + */ class ErgoApp(args: Args) extends ScorexLogging { log.info(s"Running with args: $args") diff --git a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala index 6783f5fdab..4412d63587 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala @@ -2,7 +2,7 @@ package org.ergoplatform.network import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader, ErgoSyncInfo, ErgoSyncInfoV1, ErgoSyncInfoV2} -import org.ergoplatform.nodeView.history.ErgoHistory.Height +import org.ergoplatform.nodeView.history.ErgoHistory.{Height, Time} import scorex.core.consensus.{Fork, Older, PeerChainStatus, Unknown} import scorex.core.network.ConnectedPeer import scorex.core.settings.NetworkSettings @@ -12,6 +12,9 @@ import scala.collection.mutable import scala.concurrent.duration._ import scorex.core.utils.MapPimp +/** + * Data structures and methods to keep status of peers, find ones with expired status to send sync message etc + */ final case class ErgoSyncTracker(networkSettings: NetworkSettings) extends ScorexLogging { private val MinSyncInterval: FiniteDuration = 20.seconds @@ -24,9 +27,12 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings) extends Score private[network] val statuses = mutable.Map[ConnectedPeer, ErgoPeerStatus]() + /** + * @return get all the current statuses + */ def fullInfo(): Iterable[ErgoPeerStatus] = statuses.values - private def currentTime() = System.currentTimeMillis() + private def currentTime(): Time = System.currentTimeMillis() // returns diff def updateLastSyncGetTime(peer: ConnectedPeer): Long = { @@ -38,6 +44,9 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings) extends Score currentTime - prevSyncGetTime } + /** + * @return true if sync message was sent long time ago to `peer`, or not sent at all yet + */ def notSyncedOrOutdated(peer: ConnectedPeer): Boolean = { val peerOpt = statuses.get(peer) val notSyncedOrMissing = peerOpt.forall(_.lastSyncSentTime.isEmpty) @@ -108,6 +117,9 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings) extends Score } } + /** + * Update timestamp of last sync message sent to `peer` + */ def updateLastSyncSentTime(peer: ConnectedPeer): Unit = { statuses.get(peer).foreach { status => statuses.update(peer, status.copy(lastSyncSentTime = Option(currentTime()))) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala index 8d4d1852cd..c83d82e63e 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala @@ -233,6 +233,11 @@ trait ErgoHistory object ErgoHistory extends ScorexLogging { + /** + * Type for time, represents machine-specific timestamp of a transaction + * or block section, as miliseconds passed since beginning of UNIX + * epoch on the machine + */ type Time = Long type Height = ErgoLikeContext.Height // Int @@ -272,6 +277,9 @@ object ErgoHistory extends ScorexLogging { } } + /** + * @return ErgoHistory instance with new database or database read from existing folder + */ def readOrGenerate(ergoSettings: ErgoSettings)(implicit context: ActorContext): ErgoHistory = { val db = HistoryStorage(ergoSettings) val nodeSettings = ergoSettings.nodeSettings diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala index 18d7f8f58d..5216f06662 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala @@ -156,7 +156,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score * * @return Success() if header is valid, Failure(error) otherwise */ - protected def validate(header: Header): Try[Unit] = new HeaderValidator().validate(header).toTry + protected def validate(header: Header): Try[Unit] = HeaderValidator.validate(header).toTry protected val BestHeaderKey: ByteArrayWrapper = ByteArrayWrapper(Array.fill(HashLength)(Header.modifierTypeId)) @@ -307,7 +307,7 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score } } - class HeaderValidator extends ScorexEncoding { + private object HeaderValidator extends ScorexEncoding { private def validationState: ValidationState[Unit] = ModifierValidator(ErgoValidationSettings.initial) From 0184480627686d1acaf16094ac2c790001773eeb Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Mon, 30 Jan 2023 18:15:26 +0300 Subject: [PATCH 65/90] Update README.md Co-authored-by: pragmaxim --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 053f98c195..d145829dc5 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ This client relies on some assumptions in regards with its environment: * execution environment is trusted. While seed is stored in encrypted files, and the client's wallet tries to remove secret key from memory as soon as possible when it is not needed, the - client has no any protection from side-channel attacks, memory scans etc. + client has no protection from side-channel attacks, memory scans etc. * clocks should be more or less synchronized. If timestamp of a block is more than 20 minutes in future, the block will be temporarily rejected. The client does not use NTP or other time syncing protocols. From 2e525aa7c3a93abb0cf3033594646a43a408ea32 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Tue, 31 Jan 2023 23:30:31 +0100 Subject: [PATCH 66/90] safety check --- src/main/scala/org/ergoplatform/ErgoApp.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index b0f3e3ebc0..ca0cbf6545 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -99,6 +99,11 @@ class ErgoApp(args: Args) extends ScorexLogging { None } + if(ergoSettings.nodeSettings.extraIndex) + require( + ergoSettings.nodeSettings.stateType.holdsUtxoSet && !ergoSettings.nodeSettings.isFullBlocksPruned, + "Node must store full UTXO set and all blocks to run extra indexer." + ) // Create an instance of ExtraIndexer actor (will start if "extraIndex = true" in config) ExtraIndexer(ergoSettings.chainSettings, ergoSettings.cacheSettings) From 70f5a2eabf2bd051c393235c1a9fbe7ed53d3c1d Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Thu, 2 Feb 2023 17:51:04 +0300 Subject: [PATCH 67/90] invalidate quick fix --- .../org/ergoplatform/nodeView/mempool/ErgoMemPool.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index f3d6c1df6d..34bbdc6466 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -137,8 +137,13 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, pool.get(unconfirmedTransactionId) match { case Some(utx) => invalidate(utx) case None => - log.warn(s"Can't invalidate transaction $unconfirmedTransactionId as it is not in the pool") - this + log.warn(s"pool.get failed for $unconfirmedTransactionId") + pool.orderedTransactions.valuesIterator.find(_.id == unconfirmedTransactionId) match { + case Some(utx) => invalidate(utx) + case None => + log.warn(s"Can't invalidate transaction $unconfirmedTransactionId as it is not in the pool") + this + } } } From c6095e5d2020ad2aad40e4cf74fee7d6bc4f7223 Mon Sep 17 00:00:00 2001 From: Alexander Slesarenko Date: Thu, 2 Feb 2023 16:30:36 +0100 Subject: [PATCH 68/90] upgrade sigma to v5.0.4 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 7a7dcc9d12..93ae344c07 100644 --- a/build.sbt +++ b/build.sbt @@ -36,7 +36,7 @@ val circeVersion = "0.13.0" val akkaVersion = "2.6.10" val akkaHttpVersion = "10.2.4" -val sigmaStateVersion = "5.0.3" +val sigmaStateVersion = "5.0.4" // for testing current sigmastate build (see sigmastate-ergo-it jenkins job) val effectiveSigmaStateVersion = Option(System.getenv().get("SIGMASTATE_VERSION")).getOrElse(sigmaStateVersion) From baf1c017f6e3a25f37364317096d82d9585c27c4 Mon Sep 17 00:00:00 2001 From: Alexander Slesarenko Date: Thu, 2 Feb 2023 21:43:34 +0100 Subject: [PATCH 69/90] upgrade sigma to v5.0.5-RC1 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 93ae344c07..78db953116 100644 --- a/build.sbt +++ b/build.sbt @@ -36,7 +36,7 @@ val circeVersion = "0.13.0" val akkaVersion = "2.6.10" val akkaHttpVersion = "10.2.4" -val sigmaStateVersion = "5.0.4" +val sigmaStateVersion = "5.0.5-RC1" // for testing current sigmastate build (see sigmastate-ergo-it jenkins job) val effectiveSigmaStateVersion = Option(System.getenv().get("SIGMASTATE_VERSION")).getOrElse(sigmaStateVersion) From 90e318c1f27535ea7c84ce3bcc8b8a499940fea1 Mon Sep 17 00:00:00 2001 From: Alexander Slesarenko Date: Fri, 3 Feb 2023 00:00:41 +0100 Subject: [PATCH 70/90] upgrade sigma to v5.0.5 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 78db953116..833603e31e 100644 --- a/build.sbt +++ b/build.sbt @@ -36,7 +36,7 @@ val circeVersion = "0.13.0" val akkaVersion = "2.6.10" val akkaHttpVersion = "10.2.4" -val sigmaStateVersion = "5.0.5-RC1" +val sigmaStateVersion = "5.0.5" // for testing current sigmastate build (see sigmastate-ergo-it jenkins job) val effectiveSigmaStateVersion = Option(System.getenv().get("SIGMASTATE_VERSION")).getOrElse(sigmaStateVersion) From 6e9d2eddaaaa5af5d7d0a2a7ecf2663fc0dd4b9d Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Fri, 3 Feb 2023 17:09:59 +0300 Subject: [PATCH 71/90] SnapshotsInfo, SnapshotsInfoSpecification --- .../nodeView/state/ErgoStateReader.scala | 19 ++++++++-- .../nodeView/state/SnapshotsInfo.scala | 36 +++++++++++++++++++ .../nodeView/state/UtxoState.scala | 26 ++++++++++++-- .../state/SnapshotsInfoSpecification.scala | 25 +++++++++++++ 4 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala create mode 100644 src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala index 7da217fc7c..d86d3332f4 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala @@ -10,8 +10,17 @@ import scorex.util.ScorexLogging trait ErgoStateReader extends NodeViewComponent with ScorexLogging { + /** + * Root hash and height of AVL+ tree authenticating UTXO set + */ def rootHash: ADDigest + /** + * Current version of the state + * Must be ID of last block applied + */ + def version: VersionTag + val store: LDBVersionedStore val constants: StateConstants @@ -19,6 +28,13 @@ trait ErgoStateReader extends NodeViewComponent with ScorexLogging { protected lazy val votingSettings: VotingSettings = chainSettings.voting + /** + * If the state is in its genesis version (before genesis block) + */ + def isGenesis: Boolean = { + rootHash.sameElements(constants.settings.chainSettings.genesisStateDigest) + } + def stateContext: ErgoStateContext = ErgoStateReader.storageStateContext(store, constants) /** @@ -28,9 +44,6 @@ trait ErgoStateReader extends NodeViewComponent with ScorexLogging { def genesisBoxes: Seq[ErgoBox] = ErgoState.genesisBoxes(chainSettings) - //must be ID of last applied modifier - def version: VersionTag - def closeStorage(): Unit = { log.warn("Closing state's store.") store.close() diff --git a/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala new file mode 100644 index 0000000000..27027fe2fc --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala @@ -0,0 +1,36 @@ +package org.ergoplatform.nodeView.state + +import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.nodeView.state.UtxoState.ManifestId + +/** + * Container for UTXO set snapshots the node holds + * @param availableManifests - available UTXO set snapshot manifests and corresponding heights + */ +case class SnapshotsInfo(availableManifests: Map[Height, ManifestId]) { + + /** + * @return new container instance with new snapshot added + */ + def withNewManifest(height: Height, manifestId: ManifestId): SnapshotsInfo = { + SnapshotsInfo(availableManifests.updated(height, manifestId)) + } + + /** + * @return whether snapshots available + */ + def nonEmpty: Boolean = availableManifests.nonEmpty +} + +object SnapshotsInfo { + + /** + * @return create container with no snapshots there + */ + def makeEmpty(): SnapshotsInfo = SnapshotsInfo(Map.empty) + +} + + + + diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala index b0a5e6d7ee..07d7cf4a3a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala @@ -17,6 +17,7 @@ import scorex.core.transaction.state.TransactionValidation import scorex.core.utils.ScorexEncoding import scorex.core.validation.{ModifierValidator} import scorex.crypto.authds.avltree.batch._ +import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, BatchAVLProverSubtree} import scorex.crypto.authds.{ADDigest, ADValue} import scorex.crypto.hash.Digest32 import scorex.db.{ByteArrayWrapper, LDBVersionedStore} @@ -41,12 +42,12 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 with UtxoStateReader with ScorexEncoding { + import UtxoState.metadata + override def rootHash: ADDigest = persistentProver.synchronized { persistentProver.digest } - import UtxoState.metadata - override def rollbackTo(version: VersionTag): Try[UtxoState] = persistentProver.synchronized { val p = persistentProver log.info(s"Rollback UtxoState to version ${Algos.encoder.encode(version)}") @@ -235,6 +236,27 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 object UtxoState { + /** + * Short synonym for AVL+ tree type used in the node + */ + type Manifest = BatchAVLProverManifest[Digest32] + + /** + * Short synonym for AVL subtree type used in the node + */ + type Subtree = BatchAVLProverSubtree[Digest32] + + + /** + * Manifest is associated with 32 bytes cryptographically strong unique id (root hash of the AVL tree under manifest) + */ + type ManifestId = Digest32 + + /** + * Subtree is associated with 32 bytes cryptographically strong unique id (hash of subtree's root node) + */ + type SubtreeId = Digest32 + private lazy val bestVersionKey = Algos.hash("best state version") val EmissionBoxIdKey: Digest32 = Algos.hash("emission box id key") diff --git a/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala new file mode 100644 index 0000000000..e2b589d993 --- /dev/null +++ b/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala @@ -0,0 +1,25 @@ +package org.ergoplatform.nodeView.state + +import org.ergoplatform.utils.ErgoPropertyTest + +class SnapshotsInfoSpecification extends ErgoPropertyTest { + + property("makeEmpty / nonEmpty / withNewManifest") { + val empty = SnapshotsInfo.makeEmpty() + empty.nonEmpty shouldBe false + + val h = 10 + val d = digest32Gen.sample.get + val nonEmpty = empty.withNewManifest(h, d) + nonEmpty.nonEmpty shouldBe true + nonEmpty.availableManifests(h).sameElements(d) shouldBe true + + val h2 = 20 + val d2 = digest32Gen.sample.get + val ne2 = nonEmpty.withNewManifest(h2, d2) + nonEmpty.availableManifests.size shouldBe 1 + ne2.availableManifests(h).sameElements(d) shouldBe true + ne2.availableManifests(h2).sameElements(d2) shouldBe true + } + +} From 1bdf8234ca41d5e7676ec1d1e6ed7f6b7b6ecdee Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Mon, 6 Feb 2023 19:16:23 +0300 Subject: [PATCH 72/90] addressing review comments --- .../local/ErgoStatsCollector.scala | 2 +- .../nodeView/ErgoNodeViewHolder.scala | 2 +- .../nodeView/state/DigestState.scala | 6 +++--- .../nodeView/state/ErgoState.scala | 12 ++++++++++-- .../nodeView/state/ErgoStateReader.scala | 13 ++++++------- .../nodeView/state/SnapshotsInfo.scala | 8 ++++---- .../nodeView/state/UtxoState.scala | 4 ++-- .../local/MempoolAuditorSpec.scala | 2 +- .../state/DigestStateSpecification.scala | 18 +++++++++--------- .../state/ErgoStateSpecification.scala | 8 ++++---- .../state/SnapshotsInfoSpecification.scala | 2 +- .../state/UtxoStateSpecification.scala | 8 ++++---- .../state/wrapped/WrappedDigestState.scala | 2 +- .../viewholder/ErgoNodeViewHolderSpec.scala | 18 +++++++++--------- .../ergoplatform/sanity/ErgoSanityDigest.scala | 2 +- .../ergoplatform/utils/NodeViewTestOps.scala | 2 +- .../scala/org/ergoplatform/utils/Stubs.scala | 2 +- 17 files changed, 59 insertions(+), 52 deletions(-) diff --git a/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala b/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala index 8756d159aa..febc4eebb3 100644 --- a/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala +++ b/src/main/scala/org/ergoplatform/local/ErgoStatsCollector.scala @@ -87,7 +87,7 @@ class ErgoStatsCollector(readersHolder: ActorRef, headersScore = h.bestHeaderOpt.flatMap(m => h.scoreOf(m.id)), fullBlocksScore = h.bestFullBlockOpt.flatMap(m => h.scoreOf(m.id)), genesisBlockIdOpt = h.headerIdsAtHeight(ErgoHistory.GenesisHeight).headOption, - stateRoot = Some(Algos.encode(s.rootHash)), + stateRoot = Some(Algos.encode(s.rootDigest)), stateVersion = Some(s.version), parameters = s.stateContext.currentParameters ) diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index 153b397fe7..5e13ab0d61 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -537,7 +537,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti ErgoState.readOrGenerate(settings, constants) .asInstanceOf[State] .ensuring( - state => java.util.Arrays.equals(state.rootHash, settings.chainSettings.genesisStateDigest), + state => java.util.Arrays.equals(state.rootDigest, settings.chainSettings.genesisStateDigest), "State root is incorrect" ) } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala index a58e55b445..d4e1f27595 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala @@ -26,7 +26,7 @@ import scala.util.{Failure, Success, Try} * See https://eprint.iacr.org/2016/994 for details on this mode. */ class DigestState protected(override val version: VersionTag, - override val rootHash: ADDigest, + override val rootDigest: ADDigest, override val store: LDBVersionedStore, ergoSettings: ErgoSettings) extends ErgoState[DigestState] @@ -48,7 +48,7 @@ class DigestState protected(override val version: VersionTag, val knownBoxesTry = ErgoState.stateChanges(transactions).map { stateChanges => val boxesFromProofs: Seq[ErgoBox] = - proofs.verify(stateChanges, rootHash, expectedHash).get.map(v => ErgoBoxSerializer.parseBytes(v)) + proofs.verify(stateChanges, rootDigest, expectedHash).get.map(v => ErgoBoxSerializer.parseBytes(v)) (transactions.flatMap(_.outputs) ++ boxesFromProofs).map(o => (ByteArrayWrapper(o.id), o)).toMap } @@ -107,7 +107,7 @@ class DigestState protected(override val version: VersionTag, private def processFullBlock: ModifierProcessing[DigestState] = { case fb: ErgoFullBlock if nodeSettings.verifyTransactions => log.info(s"Got new full block ${fb.encodedId} at height ${fb.header.height} with root " + - s"${Algos.encode(fb.header.stateRoot)}. Our root is ${Algos.encode(rootHash)}") + s"${Algos.encode(fb.header.stateRoot)}. Our root is ${Algos.encode(rootDigest)}") validate(fb) .flatMap { _ => val version: VersionTag = idToVersion(fb.header.id) diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala index 327cca2242..cfc95bba4b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala @@ -64,6 +64,14 @@ trait ErgoState[IState <: ErgoState[IState]] extends ErgoStateReader { */ def getReader: ErgoStateReader = this + /** + * Close database where state-related data lives + */ + def closeStorage(): Unit = { + log.warn("Closing state's store.") + store.close() + } + } object ErgoState extends ScorexLogging { @@ -262,8 +270,8 @@ object ErgoState extends ScorexLogging { val bh = BoxHolder(boxes) UtxoState.fromBoxHolder(bh, boxes.headOption, stateDir, constants, LaunchParameters).ensuring(us => { - log.info(s"Genesis UTXO state generated with hex digest ${Base16.encode(us.rootHash)}") - java.util.Arrays.equals(us.rootHash, constants.settings.chainSettings.genesisStateDigest) && us.version == genesisStateVersion + log.info(s"Genesis UTXO state generated with hex digest ${Base16.encode(us.rootDigest)}") + java.util.Arrays.equals(us.rootDigest, constants.settings.chainSettings.genesisStateDigest) && us.version == genesisStateVersion }) -> bh } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala index d86d3332f4..9479fe4cf5 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala @@ -8,12 +8,16 @@ import scorex.crypto.hash.Digest32 import scorex.db.LDBVersionedStore import scorex.util.ScorexLogging +/** + * State-related data and functions related to any state implementation ("utxo" or "digest") which are + * not modifying the state (so only reading it) + */ trait ErgoStateReader extends NodeViewComponent with ScorexLogging { /** * Root hash and height of AVL+ tree authenticating UTXO set */ - def rootHash: ADDigest + def rootDigest: ADDigest /** * Current version of the state @@ -32,7 +36,7 @@ trait ErgoStateReader extends NodeViewComponent with ScorexLogging { * If the state is in its genesis version (before genesis block) */ def isGenesis: Boolean = { - rootHash.sameElements(constants.settings.chainSettings.genesisStateDigest) + rootDigest.sameElements(constants.settings.chainSettings.genesisStateDigest) } def stateContext: ErgoStateContext = ErgoStateReader.storageStateContext(store, constants) @@ -44,11 +48,6 @@ trait ErgoStateReader extends NodeViewComponent with ScorexLogging { def genesisBoxes: Seq[ErgoBox] = ErgoState.genesisBoxes(chainSettings) - def closeStorage(): Unit = { - log.warn("Closing state's store.") - store.close() - } - } object ErgoStateReader extends ScorexLogging { diff --git a/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala index 27027fe2fc..6a57e76c50 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala @@ -7,13 +7,13 @@ import org.ergoplatform.nodeView.state.UtxoState.ManifestId * Container for UTXO set snapshots the node holds * @param availableManifests - available UTXO set snapshot manifests and corresponding heights */ -case class SnapshotsInfo(availableManifests: Map[Height, ManifestId]) { +class SnapshotsInfo(val availableManifests: Map[Height, ManifestId]) { /** * @return new container instance with new snapshot added */ def withNewManifest(height: Height, manifestId: ManifestId): SnapshotsInfo = { - SnapshotsInfo(availableManifests.updated(height, manifestId)) + new SnapshotsInfo(availableManifests.updated(height, manifestId)) } /** @@ -25,9 +25,9 @@ case class SnapshotsInfo(availableManifests: Map[Height, ManifestId]) { object SnapshotsInfo { /** - * @return create container with no snapshots there + * @return empty container with no snapshots */ - def makeEmpty(): SnapshotsInfo = SnapshotsInfo(Map.empty) + val empty: SnapshotsInfo = new SnapshotsInfo(Map.empty) } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala index 07d7cf4a3a..4265a6dfda 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala @@ -44,7 +44,7 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 import UtxoState.metadata - override def rootHash: ADDigest = persistentProver.synchronized { + override def rootDigest: ADDigest = persistentProver.synchronized { persistentProver.digest } @@ -132,7 +132,7 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 log.debug(s"Trying to apply full block with header ${fb.header.encodedId} at height $height") - val inRoot = rootHash + val inRoot = rootDigest val stateTry = stateContext.appendFullBlock(fb).flatMap { newStateContext => val txsTry = applyTransactions(fb.blockTransactions.txs, fb.header.id, fb.header.stateRoot, newStateContext) diff --git a/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala b/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala index e19c32782c..09974da381 100644 --- a/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala +++ b/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala @@ -49,7 +49,7 @@ class MempoolAuditorSpec extends AnyFlatSpec with NodeViewTestOps with ErgoTestH } .get applyBlock(genesis) shouldBe 'success - getRootHash shouldBe Algos.encode(wusAfterGenesis.rootHash) + getRootHash shouldBe Algos.encode(wusAfterGenesis.rootDigest) val boxes = ErgoState.newBoxes(genesis.transactions).find(_.ergoTree == Constants.TrueLeaf) boxes.nonEmpty shouldBe true diff --git a/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala index aa5eae981c..4d670365d1 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala @@ -21,19 +21,19 @@ class DigestStateSpecification extends ErgoPropertyTest { val fb = validFullBlock(parentOpt = None, us, bh) val dir2 = createTempDir - val ds = DigestState.create(Some(us.version), Some(us.rootHash), dir2, stateConstants) + val ds = DigestState.create(Some(us.version), Some(us.rootDigest), dir2, stateConstants) ds.applyModifier(fb, None)(_ => ()) shouldBe 'success ds.close() val state = DigestState.create(None, None, dir2, stateConstants) state.version shouldEqual fb.header.id - state.rootHash shouldEqual fb.header.stateRoot + state.rootDigest shouldEqual fb.header.stateRoot } } property("validate() - valid block") { var (us, bh) = createUtxoState(parameters) - var ds = createDigestState(us.version, us.rootHash, parameters) + var ds = createDigestState(us.version, us.rootDigest, parameters) var parentOpt: Option[ErgoFullBlock] = None forAll { seed: Int => @@ -61,7 +61,7 @@ class DigestStateSpecification extends ErgoPropertyTest { val block = validFullBlock(parentOpt = None, us, bh) block.blockTransactions.transactions.exists(_.dataInputs.nonEmpty) shouldBe true - val ds = createDigestState(us.version, us.rootHash, parameters) + val ds = createDigestState(us.version, us.rootDigest, parameters) ds.applyModifier(block, None)(_ => ()) shouldBe 'success } } @@ -80,7 +80,7 @@ class DigestStateSpecification extends ErgoPropertyTest { val block = validFullBlock(parentOpt = None, us, bh) - val ds = createDigestState(us.version, us.rootHash, parameters) + val ds = createDigestState(us.version, us.rootDigest, parameters) ds.rollbackVersions.size shouldEqual 1 @@ -90,21 +90,21 @@ class DigestStateSpecification extends ErgoPropertyTest { ds2.stateContext.lastHeaders.size shouldEqual 1 - java.util.Arrays.equals(ds2.rootHash, ds.rootHash) shouldBe false + java.util.Arrays.equals(ds2.rootDigest, ds.rootDigest) shouldBe false val ds3 = ds2.rollbackTo(ds.version).get - ds3.rootHash shouldBe ds.rootHash + ds3.rootDigest shouldBe ds.rootDigest ds3.stateContext.lastHeaders.size shouldEqual 0 - ds3.applyModifier(block, None)(_ => ()).get.rootHash shouldBe ds2.rootHash + ds3.applyModifier(block, None)(_ => ()).get.rootDigest shouldBe ds2.rootDigest } } property("validateTransactions() - dataInputs") { forAll(boxesHolderGen) { bh => val us = createUtxoState(bh, parameters) - val ds = createDigestState(us.version, us.rootHash, parameters) + val ds = createDigestState(us.version, us.rootDigest, parameters) // generate 2 independent transactions spending state boxes only val headTx = validTransactionsFromBoxes(1, bh.boxes.take(10).values.toSeq, new RandomWrapper())._1.head diff --git a/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala index caf2623f41..60d49ff68a 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala @@ -20,7 +20,7 @@ class ErgoStateSpecification extends ErgoPropertyTest { property("applyModifier() - double spending") { forAll(boxesHolderGen, Gen.choose(1: Byte, 2: Byte)) { case (bh, version) => val us = createUtxoState(bh, parameters) - val ds = createDigestState(bytesToVersion(Array.fill(32)(100: Byte)), us.rootHash, parameters) + val ds = createDigestState(bytesToVersion(Array.fill(32)(100: Byte)), us.rootDigest, parameters) val validBlock = validFullBlock(None, us, bh) val dsTxs = validBlock.transactions ++ validBlock.transactions @@ -52,7 +52,7 @@ class ErgoStateSpecification extends ErgoPropertyTest { } var (us, bh) = createUtxoState(parameters) - var ds = createDigestState(us.version, us.rootHash, parameters) + var ds = createDigestState(us.version, us.rootDigest, parameters) var lastBlocks: Seq[ErgoFullBlock] = Seq() forAll { seed: Int => val blBh = validFullBlockWithBoxHolder(lastBlocks.headOption, us, bh, new RandomWrapper(Some(seed))) @@ -68,8 +68,8 @@ class ErgoStateSpecification extends ErgoPropertyTest { property("generateGenesisUtxoState & generateGenesisDigestState are compliant") { val settings = ErgoSettings.read(Args.empty) val dir = createTempDir - val rootHash = createUtxoState(parameters)._1.rootHash - val expectedRootHash = ErgoState.generateGenesisDigestState(dir, settings).rootHash + val rootHash = createUtxoState(parameters)._1.rootDigest + val expectedRootHash = ErgoState.generateGenesisDigestState(dir, settings).rootDigest rootHash shouldBe expectedRootHash } diff --git a/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala index e2b589d993..4f3b0ffe79 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsInfoSpecification.scala @@ -5,7 +5,7 @@ import org.ergoplatform.utils.ErgoPropertyTest class SnapshotsInfoSpecification extends ErgoPropertyTest { property("makeEmpty / nonEmpty / withNewManifest") { - val empty = SnapshotsInfo.makeEmpty() + val empty = SnapshotsInfo.empty empty.nonEmpty shouldBe false val h = 10 diff --git a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala index 8b16083d58..205caa634a 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala @@ -495,7 +495,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera bh.sortedBoxes.foreach(box => us.boxById(box.id) should not be None) val genesis = validFullBlock(parentOpt = None, us, bh) val wusAfterGenesis = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get - wusAfterGenesis.rootHash shouldEqual genesis.header.stateRoot + wusAfterGenesis.rootDigest shouldEqual genesis.header.stateRoot val (finalState: WrappedUtxoState, chain: Seq[ErgoFullBlock]) = (0 until depth) .foldLeft((wusAfterGenesis, Seq(genesis))) { (sb, _) => @@ -503,17 +503,17 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera val block = validFullBlock(parentOpt = Some(sb._2.last), state) (state.applyModifier(block)(_ => ()).get, sb._2 ++ Seq(block)) } - val finalRoot = finalState.rootHash + val finalRoot = finalState.rootDigest finalRoot shouldEqual chain.last.header.stateRoot val rollbackedState = finalState.rollbackTo(idToVersion(genesis.id)).get - rollbackedState.rootHash shouldEqual genesis.header.stateRoot + rollbackedState.rootDigest shouldEqual genesis.header.stateRoot val finalState2: WrappedUtxoState = chain.tail.foldLeft(rollbackedState) { (state, block) => state.applyModifier(block)(_ => ()).get } - finalState2.rootHash shouldEqual finalRoot + finalState2.rootDigest shouldEqual finalRoot } } } diff --git a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala index d9c8abc729..fe1933fad9 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala @@ -12,7 +12,7 @@ import scala.util.Try class WrappedDigestState(val digestState: DigestState, val wrappedUtxoState: WrappedUtxoState, val settings: ErgoSettings) - extends DigestState(digestState.version, digestState.rootHash, digestState.store, settings) { + extends DigestState(digestState.version, digestState.rootDigest, digestState.store, settings) { override def applyModifier(mod: BlockSection, estimatedTip: Option[Height]) (generate: LocallyGeneratedModifier => Unit): Try[WrappedDigestState] = { diff --git a/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala b/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala index 2a49a97174..f3b89434a5 100644 --- a/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala @@ -41,7 +41,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t1 = TestCase("check genesis state") { fixture => import fixture._ - getCurrentState.rootHash shouldBe getGenesisStateDigest + getCurrentState.rootDigest shouldBe getGenesisStateDigest } private val t2 = TestCase("check history after genesis") { fixture => @@ -171,7 +171,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi applyBlock(block) shouldBe 'success getBestHeaderOpt shouldBe Some(block.header) if (verifyTransactions) { - getRootHash shouldBe Algos.encode(wusAfterBlock.rootHash) + getRootHash shouldBe Algos.encode(wusAfterBlock.rootDigest) } getBestHeaderOpt shouldBe Some(block.header) @@ -183,7 +183,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi applyBlock(brokenBlock2) shouldBe 'success getBestFullBlockOpt shouldBe Some(block) - getRootHash shouldBe Algos.encode(wusAfterBlock.rootHash) + getRootHash shouldBe Algos.encode(wusAfterBlock.rootDigest) getBestHeaderOpt shouldBe Some(block.header) } } @@ -218,7 +218,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi }.get applyBlock(genesis) shouldBe 'success - getRootHash shouldBe Algos.encode(wusAfterGenesis.rootHash) + getRootHash shouldBe Algos.encode(wusAfterGenesis.rootDigest) val chain1block1 = validFullBlock(Some(genesis), wusAfterGenesis) val expectedBestFullBlockOpt = if (verifyTransactions) Some(chain1block1) else None @@ -233,7 +233,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi val wusChain2Block1 = wusAfterGenesis.applyModifier(chain2block1)(mod => nodeViewHolderRef ! mod).get val chain2block2 = validFullBlock(Some(chain2block1), wusChain2Block1) - chain2block1.header.stateRoot shouldEqual wusChain2Block1.rootHash + chain2block1.header.stateRoot shouldEqual wusChain2Block1.rootDigest applyBlock(chain2block2) shouldBe 'success if (verifyTransactions) { @@ -292,7 +292,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi }.get applyBlock(genesis) shouldBe 'success - getRootHash shouldBe Algos.encode(wusAfterGenesis.rootHash) + getRootHash shouldBe Algos.encode(wusAfterGenesis.rootDigest) val chain2block1 = validFullBlock(Some(genesis), wusAfterGenesis) val wusChain2Block1 = wusAfterGenesis.applyModifier(chain2block1)(mod => nodeViewHolderRef ! mod).get @@ -456,7 +456,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi val wusChain2Block1 = wusGenesis.applyModifier(chain2block1)(mod => nodeViewHolderRef ! mod).get val chain2block2 = validFullBlock(Some(chain2block1), wusChain2Block1) - chain2block1.header.stateRoot shouldEqual wusChain2Block1.rootHash + chain2block1.header.stateRoot shouldEqual wusChain2Block1.rootDigest applyBlock(chain2block2) shouldBe 'success if (verifyTransactions) { @@ -508,13 +508,13 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi if (verifyTransactions) { - val initDigest = getCurrentState.rootHash + val initDigest = getCurrentState.rootDigest applyBlock(invalidBlock) shouldBe 'success getBestFullBlockOpt shouldBe None getBestHeaderOpt shouldBe None - getCurrentState.rootHash shouldEqual initDigest + getCurrentState.rootDigest shouldEqual initDigest } } diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala index db3bc448dd..9616dd81cf 100644 --- a/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala +++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala @@ -28,7 +28,7 @@ class ErgoSanityDigest extends ErgoSanity[DIGEST_ST] { override val stateGen: Gen[WrappedDigestState] = { boxesHolderGen.map(WrappedUtxoState(_, createTempDir, None, parameters, settings)).map { wus => - val digestState = DigestState.create(Some(wus.version), Some(wus.rootHash), createTempDir, stateConstants) + val digestState = DigestState.create(Some(wus.version), Some(wus.rootDigest), createTempDir, stateConstants) new WrappedDigestState(digestState, wus, settings) } } diff --git a/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala b/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala index 116bfcf0b1..d78b852d81 100644 --- a/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala +++ b/src/test/scala/org/ergoplatform/utils/NodeViewTestOps.scala @@ -132,7 +132,7 @@ trait NodeViewTestOps extends NodeViewBaseOps { def getPoolSize(implicit ctx: Ctx): Int = getCurrentView.pool.size - def getRootHash(implicit ctx: Ctx): String = Algos.encode(getCurrentState.rootHash) + def getRootHash(implicit ctx: Ctx): String = Algos.encode(getCurrentState.rootDigest) def getBestFullBlockOpt(implicit ctx: Ctx): Option[ErgoFullBlock] = getHistory.bestFullBlockOpt diff --git a/src/test/scala/org/ergoplatform/utils/Stubs.scala b/src/test/scala/org/ergoplatform/utils/Stubs.scala index d76397e9e7..66ffffeea1 100644 --- a/src/test/scala/org/ergoplatform/utils/Stubs.scala +++ b/src/test/scala/org/ergoplatform/utils/Stubs.scala @@ -58,7 +58,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with val digestState: DigestState = { boxesHolderGen.map(WrappedUtxoState(_, createTempDir, None, parameters, settings)).map { wus => - DigestState.create(Some(wus.version), Some(wus.rootHash), createTempDir, stateConstants) + DigestState.create(Some(wus.version), Some(wus.rootDigest), createTempDir, stateConstants) } }.sample.value From 05662a388b6d31f5c4e0de617b43f94db0949558 Mon Sep 17 00:00:00 2001 From: Alexander Slesarenko Date: Tue, 7 Feb 2023 21:57:23 +0100 Subject: [PATCH 73/90] configure build.sbt for Scala 2.13 --- build.sbt | 3 ++- ergo-wallet/build.sbt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 833603e31e..18b62c9859 100644 --- a/build.sbt +++ b/build.sbt @@ -6,6 +6,7 @@ logLevel := Level.Debug // this values should be in sync with ergo-wallet/build.sbt val scala211 = "2.11.12" val scala212 = "2.12.10" +val scala213 = "2.13.8" lazy val commonSettings = Seq( organization := "org.ergoplatform", @@ -264,7 +265,7 @@ lazy val avldb_benchmarks = (project in file("avldb/benchmarks")) lazy val ergoWallet = (project in file("ergo-wallet")) .disablePlugins(ScapegoatSbtPlugin) // not compatible with crossScalaVersions .settings( - crossScalaVersions := Seq(scalaVersion.value, scala211), + crossScalaVersions := Seq(scala213, scalaVersion.value, scala211), commonSettings, name := "ergo-wallet", libraryDependencies ++= Seq( diff --git a/ergo-wallet/build.sbt b/ergo-wallet/build.sbt index dcb251032e..e9dcf17c62 100644 --- a/ergo-wallet/build.sbt +++ b/ergo-wallet/build.sbt @@ -1,12 +1,13 @@ // this values should be in sync with root (i.e. ../build.sbt) val scala211 = "2.11.12" val scala212 = "2.12.10" +val scala213 = "2.13.8" val circeVersion = "0.13.0" val circeVersion211 = "0.10.0" libraryDependencies ++= Seq( - "org.scodec" %% "scodec-bits" % "1.1.6", + "org.scodec" %% "scodec-bits" % "1.1.34", "io.circe" %% "circe-core" % (if (scalaVersion.value == scala211) circeVersion211 else circeVersion), "io.circe" %% "circe-generic" % (if (scalaVersion.value == scala211) circeVersion211 else circeVersion), From 7007e7991235be52d429a662287a4e3ce3a8f2e1 Mon Sep 17 00:00:00 2001 From: Alexander Slesarenko Date: Tue, 7 Feb 2023 22:01:28 +0100 Subject: [PATCH 74/90] fix 2.13 compilation errors --- .../wallet/interface4j/crypto/ErgoUnsafeProver.java | 4 ++-- .../org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala | 2 +- .../org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala | 4 +++- .../scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala | 4 ++-- .../ergoplatform/wallet/transactions/TransactionBuilder.scala | 2 +- .../scala/org/ergoplatform/wallet/mnemonic/MnemonicSpec.scala | 2 +- .../test/scala/org/ergoplatform/wallet/utils/Generators.scala | 1 - 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java b/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java index 6f1f7dda5d..a364a07352 100644 --- a/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java +++ b/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java @@ -2,7 +2,7 @@ import org.ergoplatform.ErgoLikeTransaction; import org.ergoplatform.UnsignedErgoLikeTransaction; -import scala.collection.JavaConversions; +import scala.collection.JavaConverters; import sigmastate.basics.DLogProtocol; import java.util.Map; @@ -27,7 +27,7 @@ public ErgoLikeTransaction prove(UnsignedErgoLikeTransaction unsignedTx, DLogPro */ public ErgoLikeTransaction prove(UnsignedErgoLikeTransaction unsignedTx, Map sks) { // JavaConversions used to support Scala 2.11 - return org.ergoplatform.wallet.interpreter.ErgoUnsafeProver.prove(unsignedTx, JavaConversions.mapAsScalaMap(sks)); + return org.ergoplatform.wallet.interpreter.ErgoUnsafeProver.prove(unsignedTx, JavaConverters.mapAsScalaMap(sks)); } } diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala index cdc5f07c2e..534f55dc49 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala @@ -104,7 +104,7 @@ class DefaultBoxSelector(override val reemissionDataOpt: Option[ReemissionData]) assetsMet )) { formChangeBoxes(currentBalance, targetBalance, currentAssets, targetAssets).mapRight { changeBoxes => - selectionResultWithEip27Output(res, changeBoxes) + selectionResultWithEip27Output(res.toSeq, changeBoxes) } } else { Left(NotEnoughTokensError( diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala index 9d4a1f1330..e7e55d5288 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala @@ -3,6 +3,8 @@ package org.ergoplatform.wallet.boxes import org.ergoplatform.ErgoBoxCandidate import sigmastate.eval.Extensions._ import java7.compat.Math + +import scala.collection.compat.immutable.ArraySeq import scala.collection.mutable import scala.util.Try @@ -29,7 +31,7 @@ object ErgoBoxAssetExtractor { ) box.additionalTokens.foreach { case (assetId, amount) => - val aiWrapped = mutable.WrappedArray.make(assetId) + val aiWrapped = ArraySeq.unsafeWrapArray(assetId) val total = map.getOrElse(aiWrapped, 0L) map.put(aiWrapped, Math.addExact(total, amount)) } diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala index 700e5f48af..88db0570c4 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala @@ -66,8 +66,8 @@ object Mnemonic { * Converts mnemonic phrase to seed it was derived from. */ def toSeed(mnemonic: SecretString, passOpt: Option[SecretString] = None): Array[Byte] = { - val normalizedMnemonic = normalize(mnemonic.getData(), NFKD).toCharArray - val normalizedPass = normalize(("mnemonic".toCharArray ++ passOpt.fold("".toCharArray())(_.getData())), NFKD) + val normalizedMnemonic = normalize(ArrayCharSequence(mnemonic.getData()), NFKD).toCharArray + val normalizedPass = normalize(ArrayCharSequence("mnemonic".toCharArray ++ passOpt.fold("".toCharArray())(_.getData())), NFKD) passOpt.fold(())(_.erase()) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala index 0fbde5068d..e8dedb0c9e 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala @@ -234,7 +234,7 @@ object TransactionBuilder { // add burnTokens to target assets so that they are excluded from the change outputs // thus total outputs assets will be reduced which is interpreted as _token burning_ - val tokensOutWithBurned = AssetUtils.mergeAssets(tokensOutNoMinted, burnTokens) + val tokensOutWithBurned = AssetUtils.mergeAssets(tokensOutNoMinted.toMap, burnTokens) val selection = boxSelector.select(inputs.toIterator, outputTotal, tokensOutWithBurned) match { case Left(err) => throw new IllegalArgumentException( diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/mnemonic/MnemonicSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/mnemonic/MnemonicSpec.scala index 19a4a339d0..60db9060c8 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/mnemonic/MnemonicSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/mnemonic/MnemonicSpec.scala @@ -404,7 +404,7 @@ class MnemonicSpec val strength = Mnemonic.AllowedStrengths.zip(Mnemonic.MnemonicSentenceSizes).find(_._2 == sentenceSize).map(_._1).get val mnemonic = new Mnemonic(langId, strength) Base16.encode(Mnemonic.toSeed(SecretString.create(sentence), Some(SecretString.create(pass)))) shouldEqual seed - normalize(mnemonic.toMnemonic(Base16.decode(entropy).get).get.getData(), NFKD) shouldEqual normalize(sentence, NFKD) + normalize(ArrayCharSequence(mnemonic.toMnemonic(Base16.decode(entropy).get).get.getData()), NFKD) shouldEqual normalize(sentence, NFKD) } } diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala index 9fc73b1398..ec4f7ff88f 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala @@ -15,7 +15,6 @@ import sigmastate.basics.DLogProtocol.ProveDlog import sigmastate.{SByte, SType} import org.ergoplatform.wallet.Constants.{ScanId, PaymentsScanId} import scorex.util._ -import scala.collection.IndexedSeq import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBoxCandidate import org.ergoplatform.ErgoScriptPredef From 8ab18cdc996b9a8e98f714d164dc4134cd12bd79 Mon Sep 17 00:00:00 2001 From: Alexander Slesarenko Date: Tue, 7 Feb 2023 22:05:53 +0100 Subject: [PATCH 75/90] add 2.13 to CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a835a2110a..17d49677ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - scala: [2.12.10, 2.11.12] + scala: [2.13.8, 2.12.10, 2.11.12] java: [adopt@1.8] runs-on: ${{ matrix.os }} steps: From 305b8b246523a2a5f258ceeac651f5d849896eec Mon Sep 17 00:00:00 2001 From: Alexander Slesarenko Date: Wed, 8 Feb 2023 11:24:46 +0100 Subject: [PATCH 76/90] fix 2.11 compilation --- .../wallet/interface4j/crypto/ErgoUnsafeProver.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java b/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java index a364a07352..4493e490e5 100644 --- a/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java +++ b/ergo-wallet/src/main/java/org/ergoplatform/wallet/interface4j/crypto/ErgoUnsafeProver.java @@ -26,8 +26,10 @@ public ErgoLikeTransaction prove(UnsignedErgoLikeTransaction unsignedTx, DLogPro * @return signed transaction */ public ErgoLikeTransaction prove(UnsignedErgoLikeTransaction unsignedTx, Map sks) { - // JavaConversions used to support Scala 2.11 - return org.ergoplatform.wallet.interpreter.ErgoUnsafeProver.prove(unsignedTx, JavaConverters.mapAsScalaMap(sks)); + // This method of JavaConverters is supported across Scala 2.11-2.13 + return org.ergoplatform.wallet.interpreter.ErgoUnsafeProver.prove( + unsignedTx, + JavaConverters.mapAsScalaMapConverter(sks).asScala()); } } From 2966717590b3314045dcf55dbd45c3562a1f07db Mon Sep 17 00:00:00 2001 From: Boris P Date: Thu, 9 Feb 2023 00:53:18 -0700 Subject: [PATCH 77/90] bump scala-sbt image to 11.0.15_1.7.1_2.13.8 Make docker build a bit faster by not having to get every single version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1c5f95e3c4..95b719cb24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM sbtscala/scala-sbt:eclipse-temurin-11.0.15_1.6.2_2.12.16 as builder +FROM sbtscala/scala-sbt:eclipse-temurin-11.0.15_1.7.1_2.13.8 as builder WORKDIR /mnt COPY build.sbt findbugs-exclude.xml ./ COPY project/ project/ From 4c5e36383de1f12020aae0a1c7537b3ba1daf354 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Thu, 9 Feb 2023 21:51:42 +0300 Subject: [PATCH 78/90] put fix --- .../nodeView/mempool/ErgoMemPool.scala | 10 ++---- .../nodeView/mempool/OrderedTxPool.scala | 35 +++++++++++-------- .../nodeView/mempool/ErgoMemPoolSpec.scala | 14 ++++++++ 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index 34bbdc6466..18cab5845f 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -88,17 +88,13 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, /** * Method to put a transaction into the memory pool. Validation of the transactions against - * the state is done in NodeVieHolder. This put() method can check whether a transaction is valid + * the state is done in NodeViewHolder. This put() method can check whether a transaction is valid * @param unconfirmedTx * @return Success(updatedPool), if transaction successfully added to the pool, Failure(_) otherwise */ def put(unconfirmedTx: UnconfirmedTransaction): ErgoMemPool = { - if (!pool.contains(unconfirmedTx.id)) { - val updatedPool = pool.put(unconfirmedTx, feeFactor(unconfirmedTx)) - new ErgoMemPool(updatedPool, stats, sortingOption) - } else { - this - } + val updatedPool = pool.put(unconfirmedTx, feeFactor(unconfirmedTx)) + new ErgoMemPool(updatedPool, stats, sortingOption) } def put(txs: TraversableOnce[UnconfirmedTransaction]): ErgoMemPool = { diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala index 30b92bcdfc..5effd80479 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala @@ -66,14 +66,26 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT */ def put(unconfirmedTx: UnconfirmedTransaction, feeFactor: Int): OrderedTxPool = { val tx = unconfirmedTx.transaction - val wtx = weighted(tx, feeFactor) - val newPool = OrderedTxPool( - orderedTransactions.updated(wtx, unconfirmedTx), - transactionsRegistry.updated(wtx.id, wtx), - invalidatedTxIds, - outputs ++ tx.outputs.map(_.id -> wtx), - inputs ++ tx.inputs.map(_.boxId -> wtx) - ).updateFamily(tx, wtx.weight, System.currentTimeMillis(), 0) + + val newPool = transactionsRegistry.get(tx.id) match { + case Some(wtx) => + OrderedTxPool( + orderedTransactions.updated(wtx, unconfirmedTx), + transactionsRegistry, + invalidatedTxIds, + outputs, + inputs + ) + case None => + val wtx = weighted(tx, feeFactor) + OrderedTxPool( + orderedTransactions.updated(wtx, unconfirmedTx), + transactionsRegistry.updated(wtx.id, wtx), + invalidatedTxIds, + outputs ++ tx.outputs.map(_.id -> wtx), + inputs ++ tx.inputs.map(_.boxId -> wtx) + ).updateFamily(tx, wtx.weight, System.currentTimeMillis(), 0) + } if (newPool.orderedTransactions.size > mempoolCapacity) { val victim = newPool.orderedTransactions.last._2 newPool.remove(victim) @@ -123,13 +135,6 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT } } - def filter(condition: UnconfirmedTransaction => Boolean): OrderedTxPool = { - orderedTransactions.foldLeft(this)((pool, entry) => { - val tx = entry._2 - if (condition(tx)) pool else pool.remove(tx) - }) - } - /** * Do not place transaction in the pool if the transaction known to be invalid, pool already has it, or the pool * is overfull. diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala index faaf40218e..34d24c09ed 100644 --- a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala @@ -378,6 +378,20 @@ class ErgoMemPoolSpec extends AnyFlatSpec pool.size shouldBe 0 pool.stats.takenTxns shouldBe (family_depth + 1) * txs.size } + + it should "put not adding transaction twice" in { + val pool = ErgoMemPool.empty(settings).pool + val tx = invalidErgoTransactionGen.sample.get + val now = System.currentTimeMillis() + + val utx1 = UnconfirmedTransaction(tx, None, now, now, None, None) + val utx2 = UnconfirmedTransaction(tx, None, now, now, None, None) + val utx3 = UnconfirmedTransaction(tx, None, now + 1, now + 1, None, None) + val updPool = pool.put(utx1, 100).remove(utx1).put(utx2, 500).put(utx3, 5000) + updPool.size shouldBe 1 + updPool.get(utx3.id).get.lastCheckedTime shouldBe (now + 1) + } + } From a6f4692f3136ecd5e6a5e5e8743153593380a37c Mon Sep 17 00:00:00 2001 From: jellymlg Date: Thu, 9 Feb 2023 23:22:43 +0100 Subject: [PATCH 79/90] further optimization --- avldb/src/main/scala/scorex/db/LDBKVStore.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/avldb/src/main/scala/scorex/db/LDBKVStore.scala b/avldb/src/main/scala/scorex/db/LDBKVStore.scala index edbc958a7e..ffd53609db 100644 --- a/avldb/src/main/scala/scorex/db/LDBKVStore.scala +++ b/avldb/src/main/scala/scorex/db/LDBKVStore.scala @@ -16,9 +16,11 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging def update(toInsert: Array[(K, V)], toRemove: Array[K]): Try[Unit] = { val batch = db.createWriteBatch() + val insertLen = toInsert.length + val removeLen = toRemove.length try { - cfor(0)(_ < toInsert.length, _ + 1) { i => batch.put(toInsert(i)._1, toInsert(i)._2)} - cfor(0)(_ < toRemove.length, _ + 1) { i => batch.delete(toRemove(i))} + cfor(0)(_ < insertLen, _ + 1) { i => batch.put(toInsert(i)._1, toInsert(i)._2)} + cfor(0)(_ < removeLen, _ + 1) { i => batch.delete(toRemove(i))} db.write(batch) Success(()) } catch { From 16168ed4213e5c6134cefee14e096bf088ca783c Mon Sep 17 00:00:00 2001 From: jellymlg Date: Thu, 9 Feb 2023 23:24:29 +0100 Subject: [PATCH 80/90] fixed api doc --- src/main/resources/api/openapi.yaml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index c7e863fda2..0bda39c3fe 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -281,7 +281,7 @@ components: type: boolean example: false globalIndex: - description: The global index of the box + description: Global index of the output in the blockchain type: integer format: int64 minimum: 0 @@ -484,7 +484,7 @@ components: format: int32 example: 3 globalIndex: - description: index of the box globally + description: Global index of the transaction in the blockchain type: integer format: int64 example: 3565445 @@ -5780,7 +5780,7 @@ paths: $ref: '#/components/schemas/IndexedErgoTransaction' total: type: integer - description: Total count of transactions + description: Total count of retreived transactions '404': description: No transactions found for wanted address content: @@ -5945,7 +5945,7 @@ paths: $ref: '#/components/schemas/IndexedErgoTransaction' total: type: integer - description: Total number of boxes + description: Total number of retreived boxes '404': description: No boxes found for wanted address content: @@ -6098,7 +6098,7 @@ paths: $ref: '#/components/schemas/IndexedErgoBox' total: type: integer - description: Total number of boxes + description: Total number of retreived boxes default: description: Error content: @@ -6143,7 +6143,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/IndexedErgoBox' + type: object + properties: + items: + type: array + description: Array of boxes + items: + $ref: '#/components/schemas/IndexedErgoBox' + total: + type: integer + description: Total number of retreived boxes '404': description: No unspent box found with wanted ergotree content: From 54eb51e4bf3e82a6bda583352245c5eaeeb943fb Mon Sep 17 00:00:00 2001 From: jellymlg Date: Thu, 9 Feb 2023 23:24:58 +0100 Subject: [PATCH 81/90] added ScalaDoc --- .../scala/org/ergoplatform/http/api/BlockchainApiRoute.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 2828400c0a..be150407b9 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -30,6 +30,9 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private val paging: Directive[(Int, Int)] = parameters("offset".as[Int] ? 0, "limit".as[Int] ? 5) + /** + * Total number of boxes/transactions that can be requested at once + */ private val MaxItems = 16384 override implicit val ergoAddressEncoder: ErgoAddressEncoder = ergoSettings.chainSettings.addressEncoder From d4c3a9ddf65b2df7951a3c84fb1e7e956fcbcfb7 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Fri, 10 Feb 2023 16:11:59 +0300 Subject: [PATCH 82/90] invalidate fix --- .../nodeView/mempool/ErgoMemPool.scala | 3 +- .../nodeView/mempool/OrderedTxPool.scala | 39 ++++++++++++------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index 18cab5845f..8e5431c369 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -135,7 +135,8 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, case None => log.warn(s"pool.get failed for $unconfirmedTransactionId") pool.orderedTransactions.valuesIterator.find(_.id == unconfirmedTransactionId) match { - case Some(utx) => invalidate(utx) + case Some(utx) => + invalidate(utx) case None => log.warn(s"Can't invalidate transaction $unconfirmedTransactionId as it is not in the pool") this diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala index 5effd80479..b51b4194c5 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala @@ -17,12 +17,12 @@ import scala.collection.immutable.TreeMap * @param outputs - mapping `box.id` -> `WeightedTxId(tx.id,tx.weight)` required for getting a transaction by its output box * @param inputs - mapping `box.id` -> `WeightedTxId(tx.id,tx.weight)` required for getting a transaction by its input box id */ -case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTransaction], - transactionsRegistry: TreeMap[ModifierId, WeightedTxId], - invalidatedTxIds: ApproximateCacheLike[String], - outputs: TreeMap[BoxId, WeightedTxId], - inputs: TreeMap[BoxId, WeightedTxId]) - (implicit settings: ErgoSettings) extends ScorexLogging { +class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTransaction], + val transactionsRegistry: TreeMap[ModifierId, WeightedTxId], + val invalidatedTxIds: ApproximateCacheLike[String], + val outputs: TreeMap[BoxId, WeightedTxId], + val inputs: TreeMap[BoxId, WeightedTxId]) + (implicit settings: ErgoSettings) extends ScorexLogging { import OrderedTxPool.weighted @@ -69,7 +69,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT val newPool = transactionsRegistry.get(tx.id) match { case Some(wtx) => - OrderedTxPool( + new OrderedTxPool( orderedTransactions.updated(wtx, unconfirmedTx), transactionsRegistry, invalidatedTxIds, @@ -78,7 +78,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT ) case None => val wtx = weighted(tx, feeFactor) - OrderedTxPool( + new OrderedTxPool( orderedTransactions.updated(wtx, unconfirmedTx), transactionsRegistry.updated(wtx.id, wtx), invalidatedTxIds, @@ -106,7 +106,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT def remove(tx: ErgoTransaction): OrderedTxPool = { transactionsRegistry.get(tx.id) match { case Some(wtx) => - OrderedTxPool( + new OrderedTxPool( orderedTransactions - wtx, transactionsRegistry - tx.id, invalidatedTxIds, @@ -123,7 +123,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT val tx = unconfirmedTx.transaction transactionsRegistry.get(tx.id) match { case Some(wtx) => - OrderedTxPool( + new OrderedTxPool( orderedTransactions - wtx, transactionsRegistry - tx.id, invalidatedTxIds.put(tx.id), @@ -131,7 +131,17 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT inputs -- tx.inputs.map(_.boxId) ).updateFamily(tx, -wtx.weight, System.currentTimeMillis(), depth = 0) case None => - OrderedTxPool(orderedTransactions, transactionsRegistry, invalidatedTxIds.put(tx.id), outputs, inputs) + if (transactionsRegistry.contains(tx.id)) { + new OrderedTxPool( + orderedTransactions, + transactionsRegistry - tx.id, + invalidatedTxIds.put(tx.id), + outputs -- tx.outputs.map(_.id), + inputs -- tx.inputs.map(_.boxId) + ) + } else { + new OrderedTxPool(orderedTransactions, transactionsRegistry, invalidatedTxIds.put(tx.id), outputs, inputs) + } } } @@ -180,13 +190,14 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT this } else { - val uniqueTxIds: Set[WeightedTxId] = tx.inputs.flatMap(input => this.outputs.get(input.boxId))(collection.breakOut) + val uniqueTxIds: Set[WeightedTxId] = tx.inputs.flatMap(input => this.outputs.get(input.boxId)).toSet val parentTxs = uniqueTxIds.flatMap(wtx => this.orderedTransactions.get(wtx).map(ut => wtx -> ut)) parentTxs.foldLeft(this) { case (pool, (wtx, ut)) => val parent = ut.transaction val newWtx = WeightedTxId(wtx.id, wtx.weight + weight, wtx.feePerFactor, wtx.created) - val newPool = OrderedTxPool(pool.orderedTransactions - wtx + (newWtx -> ut), + val newPool = new OrderedTxPool( + pool.orderedTransactions - wtx + (newWtx -> ut), pool.transactionsRegistry.updated(parent.id, newWtx), invalidatedTxIds, parent.outputs.foldLeft(pool.outputs)((newOutputs, box) => newOutputs.updated(box.id, newWtx)), @@ -225,7 +236,7 @@ object OrderedTxPool { val cacheSettings = settings.cacheSettings.mempool val frontCacheSize = cacheSettings.invalidModifiersCacheSize val frontCacheExpiration = cacheSettings.invalidModifiersCacheExpiration - OrderedTxPool( + new OrderedTxPool( TreeMap.empty[WeightedTxId, UnconfirmedTransaction], TreeMap.empty[ModifierId, WeightedTxId], ExpiringApproximateCache.empty(frontCacheSize, frontCacheExpiration), From 30c0823ca84b4c7d2f72def2892ad9af9b121664 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Fri, 10 Feb 2023 16:15:56 +0300 Subject: [PATCH 83/90] condition fix --- .../scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala index b51b4194c5..6fb24edd25 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala @@ -131,7 +131,7 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr inputs -- tx.inputs.map(_.boxId) ).updateFamily(tx, -wtx.weight, System.currentTimeMillis(), depth = 0) case None => - if (transactionsRegistry.contains(tx.id)) { + if (orderedTransactions.valuesIterator.exists(utx => utx.id == tx.id)) { new OrderedTxPool( orderedTransactions, transactionsRegistry - tx.id, From 0cdada3a8b9bb30585cd2cd2128e0998a7578b68 Mon Sep 17 00:00:00 2001 From: jellymlg Date: Sat, 11 Feb 2023 16:35:26 +0100 Subject: [PATCH 84/90] updated ScalaDoc --- .../scala/org/ergoplatform/http/api/BlockchainApiRoute.scala | 2 +- .../ergoplatform/nodeView/history/extra/IndexedErgoBox.scala | 2 +- .../nodeView/history/extra/IndexedErgoTransaction.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index be150407b9..3b79ea3b27 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -31,7 +31,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting private val paging: Directive[(Int, Int)] = parameters("offset".as[Int] ? 0, "limit".as[Int] ? 5) /** - * Total number of boxes/transactions that can be requested at once + * Total number of boxes/transactions that can be requested at once to avoid too heavy requests ([[BlocksApiRoute.MaxHeaders]]) */ private val MaxItems = 16384 diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala index 9d70620a0f..7b23ee3380 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoBox.scala @@ -15,7 +15,7 @@ import scorex.util.serialization.{Reader, Writer} * @param spendingTxIdOpt - optional, id of the spending transaction * @param spendingHeightOpt - optional, height of the block in which the spending transaction was included in * @param box - underlying ErgoBox - * @param globalIndex - numeric index of the box + * @param globalIndex - serial number of this output counting from genesis box */ class IndexedErgoBox(val inclusionHeight: Int, var spendingTxIdOpt: Option[ModifierId], diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala index aba5ff6f84..c243652466 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedErgoTransaction.scala @@ -14,7 +14,7 @@ import spire.implicits.cfor * Index of a transaction. * @param txid - id of this transaction * @param height - height of the block which includes this transaction - * @param globalIndex - numeric index of this transaction + * @param globalIndex - serial number of this transaction counting from block 1 * @param inputNums - list of transaction inputs (needed for rollback) */ case class IndexedErgoTransaction(txid: ModifierId, From 74c632f7dc82e89f8117122b523131a4e14cfe65 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Mon, 13 Feb 2023 19:32:43 +0300 Subject: [PATCH 85/90] orderedTransactions.filter --- .../http/api/ErgoBaseApiRoute.scala | 4 ++-- .../mempool/UnconfirmedTransaction.scala | 24 ++++++++++++------- .../nodeView/mempool/OrderedTxPool.scala | 2 +- .../nodeView/mempool/ErgoMemPoolSpec.scala | 6 ++--- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala index 5c00e87902..dafc0bf0a1 100644 --- a/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/ErgoBaseApiRoute.scala @@ -91,9 +91,9 @@ trait ErgoBaseApiRoute extends ApiRoute with ApiCodecs { val maxTxCost = ergoSettings.nodeSettings.maxTransactionCost utxo.withMempool(mp) .validateWithCost(tx, maxTxCost) - .map(cost => UnconfirmedTransaction(tx, Some(cost), now, now, bytes, source = None)) + .map(cost => new UnconfirmedTransaction(tx, Some(cost), now, now, bytes, source = None)) case _ => - tx.statelessValidity().map(_ => UnconfirmedTransaction(tx, None, now, now, bytes, source = None)) + tx.statelessValidity().map(_ => new UnconfirmedTransaction(tx, None, now, now, bytes, source = None)) } } diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala index 5c99a0e4e3..bd18f338d8 100644 --- a/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala +++ b/src/main/scala/org/ergoplatform/modifiers/mempool/UnconfirmedTransaction.scala @@ -13,12 +13,12 @@ import scorex.util.{ModifierId, ScorexLogging} * @param transactionBytes - transaction bytes, to avoid serializations when we send it over the wire * @param source - peer which delivered the transaction (None if transaction submitted via API) */ -case class UnconfirmedTransaction(transaction: ErgoTransaction, - lastCost: Option[Int], - createdTime: Long, - lastCheckedTime: Long, - transactionBytes: Option[Array[Byte]], - source: Option[ConnectedPeer]) +class UnconfirmedTransaction(val transaction: ErgoTransaction, + val lastCost: Option[Int], + val createdTime: Long, + val lastCheckedTime: Long, + val transactionBytes: Option[Array[Byte]], + val source: Option[ConnectedPeer]) extends ScorexLogging { def id: ModifierId = transaction.id @@ -27,7 +27,13 @@ case class UnconfirmedTransaction(transaction: ErgoTransaction, * Updates cost and last checked time of unconfirmed transaction */ def withCost(cost: Int): UnconfirmedTransaction = { - copy(lastCost = Some(cost), lastCheckedTime = System.currentTimeMillis()) + new UnconfirmedTransaction( + transaction, + lastCost = Some(cost), + createdTime, + lastCheckedTime = System.currentTimeMillis(), + transactionBytes, + source) } override def equals(obj: Any): Boolean = obj match { @@ -42,12 +48,12 @@ object UnconfirmedTransaction { def apply(tx: ErgoTransaction, source: Option[ConnectedPeer]): UnconfirmedTransaction = { val now = System.currentTimeMillis() - UnconfirmedTransaction(tx, None, now, now, Some(tx.bytes), source) + new UnconfirmedTransaction(tx, None, now, now, Some(tx.bytes), source) } def apply(tx: ErgoTransaction, txBytes: Array[Byte], source: Option[ConnectedPeer]): UnconfirmedTransaction = { val now = System.currentTimeMillis() - UnconfirmedTransaction(tx, None, now, now, Some(txBytes), source) + new UnconfirmedTransaction(tx, None, now, now, Some(txBytes), source) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala index 6fb24edd25..34b70eea53 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala @@ -133,7 +133,7 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr case None => if (orderedTransactions.valuesIterator.exists(utx => utx.id == tx.id)) { new OrderedTxPool( - orderedTransactions, + orderedTransactions.filter(_._2.id != tx.id), transactionsRegistry - tx.id, invalidatedTxIds.put(tx.id), outputs -- tx.outputs.map(_.id), diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala index 34d24c09ed..c68d279f50 100644 --- a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala @@ -384,9 +384,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec val tx = invalidErgoTransactionGen.sample.get val now = System.currentTimeMillis() - val utx1 = UnconfirmedTransaction(tx, None, now, now, None, None) - val utx2 = UnconfirmedTransaction(tx, None, now, now, None, None) - val utx3 = UnconfirmedTransaction(tx, None, now + 1, now + 1, None, None) + val utx1 = new UnconfirmedTransaction(tx, None, now, now, None, None) + val utx2 = new UnconfirmedTransaction(tx, None, now, now, None, None) + val utx3 = new UnconfirmedTransaction(tx, None, now + 1, now + 1, None, None) val updPool = pool.put(utx1, 100).remove(utx1).put(utx2, 500).put(utx3, 5000) updPool.size shouldBe 1 updPool.get(utx3.id).get.lastCheckedTime shouldBe (now + 1) From e65e14a629bc89013aae3d17b43391228cd7d2a9 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Tue, 14 Feb 2023 01:16:01 +0300 Subject: [PATCH 86/90] currentTime memoization --- .../scala/org/ergoplatform/network/ErgoSyncTracker.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala index 4412d63587..133a032615 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala @@ -36,12 +36,13 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings) extends Score // returns diff def updateLastSyncGetTime(peer: ConnectedPeer): Long = { + val now = currentTime() val prevSyncGetTime = statuses.get(peer).flatMap(_.lastSyncGetTime).getOrElse(0L) statuses.get(peer).foreach { status => - statuses.update(peer, status.copy(lastSyncGetTime = Option(currentTime()))) + statuses.update(peer, status.copy(lastSyncGetTime = Option(now))) } - currentTime - prevSyncGetTime + now - prevSyncGetTime } /** From 012fd2bb337374ee7e9aefd3c86d4e5b94f016a4 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Tue, 14 Feb 2023 01:25:04 +0300 Subject: [PATCH 87/90] rollback scaladoc --- .../org/ergoplatform/network/ErgoNodeViewSynchronizer.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 4b091327ea..dff3b03c30 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -1252,6 +1252,10 @@ object ErgoNodeViewSynchronizer { case class ChangedState(reader: ErgoStateReader) extends NodeViewChange + /** + * Event which is published when rollback happened (on finding a better chain) + * @param branchPoint - block id which is last in the chain after rollback (before applying blocks from a fork) + */ case class Rollback(branchPoint: ModifierId) extends NodeViewHolderEvent case object RollbackFailed extends NodeViewHolderEvent From 50dffbe63167c4d7b9a3588ea468b6c74ae87651 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Tue, 14 Feb 2023 15:34:43 +0300 Subject: [PATCH 88/90] peersToSyncWith optimization --- src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala index 133a032615..02332cb46d 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala @@ -189,8 +189,9 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings) extends Score unknowns } val nonOutdated = eldersAndUnknown ++ forks + val now = currentTime() nonOutdated.filter { case (_, status) => - (currentTime() - status.lastSyncSentTime.getOrElse(0L)).millis >= MinSyncInterval + (now - status.lastSyncSentTime.getOrElse(0L)).millis >= MinSyncInterval }.map(_._1) } From b637d1b536cafdadb266e7e3c01b512cc7e1f16f Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Tue, 14 Feb 2023 15:35:32 +0300 Subject: [PATCH 89/90] peersToSyncWith optimization --- src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala index 133a032615..02332cb46d 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala @@ -189,8 +189,9 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings) extends Score unknowns } val nonOutdated = eldersAndUnknown ++ forks + val now = currentTime() nonOutdated.filter { case (_, status) => - (currentTime() - status.lastSyncSentTime.getOrElse(0L)).millis >= MinSyncInterval + (now - status.lastSyncSentTime.getOrElse(0L)).millis >= MinSyncInterval }.map(_._1) } From 5ac73d545b1526c2a5a2033ed5c611998ebf4b25 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Tue, 14 Feb 2023 15:47:16 +0300 Subject: [PATCH 90/90] scaladoc for invalidate --- .../org/ergoplatform/nodeView/mempool/OrderedTxPool.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala index 34b70eea53..127edd8201 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala @@ -119,6 +119,9 @@ class OrderedTxPool(val orderedTransactions: TreeMap[WeightedTxId, UnconfirmedTr def remove(utx: UnconfirmedTransaction): OrderedTxPool = remove(utx.transaction) + /** + * Remove transaction from the pool and add it to invalidated transaction ids cache + */ def invalidate(unconfirmedTx: UnconfirmedTransaction): OrderedTxPool = { val tx = unconfirmedTx.transaction transactionsRegistry.get(tx.id) match {