diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index c901fe8fff..182ba9c4c7 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -9,7 +9,9 @@ import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.ledger.LedgerService; -import com.iota.iri.service.milestone.*; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.milestone.SeenMilestonesRetriever; import com.iota.iri.service.snapshot.LocalSnapshotManager; import com.iota.iri.service.snapshot.SnapshotException; import com.iota.iri.service.snapshot.SnapshotProvider; @@ -22,6 +24,12 @@ import com.iota.iri.service.transactionpruning.DepthPruningCondition; import com.iota.iri.service.transactionpruning.SizePruningCondition; import com.iota.iri.service.transactionpruning.TransactionPruner; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; +import com.iota.iri.storage.Indexable; +import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.PersistenceProvider; +import com.iota.iri.storage.Tangle; import com.iota.iri.storage.*; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Pair; @@ -83,10 +91,6 @@ public class Iota { public final MilestoneService milestoneService; - public final LatestMilestoneTracker latestMilestoneTracker; - - public final LatestSolidMilestoneTracker latestSolidMilestoneTracker; - public final SeenMilestonesRetriever seenMilestonesRetriever; public final LedgerService ledgerService; @@ -95,6 +99,8 @@ public class Iota { public final MilestoneSolidifier milestoneSolidifier; + public final TransactionSolidifier transactionSolidifier; + public final BundleValidator bundleValidator; public final Tangle tangle; @@ -116,17 +122,17 @@ public class Iota { * @param configuration Information about how this node will be configured. * */ + //public Iota(IotaConfig configuration, SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, LocalSnapshotManager localSnapshotManager, MilestoneService milestoneService, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector, TransactionSolidifier transactionSolidifier) { public Iota(IotaConfig configuration, SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, LocalSnapshotManager localSnapshotManager, - MilestoneService milestoneService, LatestMilestoneTracker latestMilestoneTracker, - LatestSolidMilestoneTracker latestSolidMilestoneTracker, SeenMilestonesRetriever seenMilestonesRetriever, + MilestoneService milestoneService, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector, LocalSnapshotsPersistenceProvider localSnapshotsDb, - CacheManager cacheManager) { + CacheManager cacheManager, TransactionSolidifier transactionSolidifier) { this.configuration = configuration; this.ledgerService = ledgerService; @@ -136,14 +142,13 @@ public Iota(IotaConfig configuration, SpentAddressesProvider spentAddressesProvi this.snapshotService = snapshotService; this.localSnapshotManager = localSnapshotManager; this.milestoneService = milestoneService; - this.latestMilestoneTracker = latestMilestoneTracker; - this.latestSolidMilestoneTracker = latestSolidMilestoneTracker; this.seenMilestonesRetriever = seenMilestonesRetriever; this.milestoneSolidifier = milestoneSolidifier; this.transactionPruner = transactionPruner; this.neighborRouter = neighborRouter; this.txPipeline = transactionProcessingPipeline; this.tipsRequester = tipsRequester; + this.transactionSolidifier = transactionSolidifier; this.localSnapshotsDb = localSnapshotsDb; @@ -165,7 +170,6 @@ private void initDependencies() throws SnapshotException, SpentAddressesExceptio boolean assertSpentAddressesExistence = !configuration.isTestnet() && snapshotProvider.getInitialSnapshot().getIndex() != configuration.getMilestoneStartIndex(); spentAddressesProvider.init(assertSpentAddressesExistence); - latestMilestoneTracker.init(); seenMilestonesRetriever.init(); if (transactionPruner != null) { transactionPruner.init(); @@ -199,15 +203,12 @@ public void init() throws Exception { tangle.clearMetadata(com.iota.iri.model.persistables.Transaction.class); } - transactionValidator.init(); - txPipeline.start(); neighborRouter.start(); tipsRequester.start(); - latestMilestoneTracker.start(); - latestSolidMilestoneTracker.start(); seenMilestonesRetriever.start(); + transactionSolidifier.start(); milestoneSolidifier.start(); if (localSnapshotManager != null) { @@ -215,7 +216,7 @@ public void init() throws Exception { localSnapshotManager.addPruningConditions( new DepthPruningCondition(configuration, snapshotProvider, tangle), new SizePruningCondition(tangle, configuration)); - localSnapshotManager.start(latestMilestoneTracker); + localSnapshotManager.start(milestoneSolidifier); } if (transactionPruner != null) { transactionPruner.start(); @@ -255,9 +256,8 @@ private void rescanDb() throws Exception { public void shutdown() throws Exception { // shutdown in reverse starting order (to not break any dependencies) milestoneSolidifier.shutdown(); + transactionSolidifier.shutdown(); seenMilestonesRetriever.shutdown(); - latestSolidMilestoneTracker.shutdown(); - latestMilestoneTracker.shutdown(); if (transactionPruner != null) { transactionPruner.shutdown(); @@ -269,7 +269,6 @@ public void shutdown() throws Exception { tipsRequester.shutdown(); txPipeline.shutdown(); neighborRouter.shutdown(); - transactionValidator.shutdown(); localSnapshotsDb.shutdown(); tangle.shutdown(); diff --git a/src/main/java/com/iota/iri/MainInjectionConfiguration.java b/src/main/java/com/iota/iri/MainInjectionConfiguration.java index e8347fe6f1..e995b1a511 100644 --- a/src/main/java/com/iota/iri/MainInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/MainInjectionConfiguration.java @@ -11,7 +11,9 @@ import com.iota.iri.service.API; import com.iota.iri.service.ledger.LedgerService; import com.iota.iri.service.ledger.impl.LedgerServiceImpl; -import com.iota.iri.service.milestone.*; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.milestone.SeenMilestonesRetriever; import com.iota.iri.service.milestone.impl.*; import com.iota.iri.service.snapshot.LocalSnapshotManager; import com.iota.iri.service.snapshot.SnapshotProvider; @@ -27,6 +29,9 @@ import com.iota.iri.service.tipselection.impl.*; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; +import com.iota.iri.service.validation.impl.TransactionSolidifierImpl; import com.iota.iri.storage.LocalSnapshotsPersistenceProvider; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; @@ -91,33 +96,12 @@ LedgerService provideLedgerService(Tangle tangle, SnapshotProvider snapshotProvi return new LedgerServiceImpl(tangle, snapshotProvider, snapshotService, milestoneService, spentAddressesService, bundleValidator); } - @Singleton - @Provides - LatestMilestoneTracker provideLatestMilestoneTracker(Tangle tangle, SnapshotProvider snapshotProvider, MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier) { - return new LatestMilestoneTrackerImpl(tangle, snapshotProvider, milestoneService, milestoneSolidifier, configuration); - } - - @Singleton - @Provides - LatestSolidMilestoneTracker provideLatestSolidMilestoneTracker(Tangle tangle, SnapshotProvider snapshotProvider, - MilestoneService milestoneService, LedgerService ledgerService, - LatestMilestoneTracker latestMilestoneTracker, TransactionRequester transactionRequester) { - return new LatestSolidMilestoneTrackerImpl(tangle, snapshotProvider, milestoneService, ledgerService, - latestMilestoneTracker, transactionRequester, configuration); - } - @Singleton @Provides SeenMilestonesRetriever provideSeenMilestonesRetriever(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester) { return new SeenMilestonesRetrieverImpl(tangle, snapshotProvider, transactionRequester); } - @Singleton - @Provides - MilestoneSolidifier provideMilestoneSolidifier(SnapshotProvider snapshotProvider, TransactionValidator transactionValidator) { - return new MilestoneSolidifierImpl(snapshotProvider, transactionValidator); - } - @Singleton @Provides TransactionPruner provideTransactionPruner(Tangle tangle, SnapshotProvider snapshotProvider, SpentAddressesService spentAddressesService, SpentAddressesProvider spentAddressesProvider, TipsViewModel tipsViewModel) { @@ -136,8 +120,14 @@ LocalSnapshotManager provideLocalSnapshotManager(SnapshotProvider snapshotProvid @Singleton @Provides - TransactionValidator provideTransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester) { - return new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester, configuration); + TransactionValidator provideTransactionValidator(SnapshotProvider snapshotProvider, TransactionRequester transactionRequester) { + return new TransactionValidator(snapshotProvider, transactionRequester, configuration); + } + + @Singleton + @Provides + TransactionSolidifier provideTransactionSolidifier(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester, TipsViewModel tipsViewModel){ + return new TransactionSolidifierImpl(tangle, snapshotProvider, transactionRequester, tipsViewModel, configuration.getCoordinator()); } @Singleton @@ -149,9 +139,9 @@ CacheManager provideCacheManager() { @Singleton @Provides TipSelector provideTipSelector(Tangle tangle, SnapshotProvider snapshotProvider, - LatestMilestoneTracker latestMilestoneTracker, LedgerService ledgerService) { + MilestoneSolidifier milestoneSolidifier, LedgerService ledgerService) { EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, - latestMilestoneTracker); + milestoneSolidifier); RatingCalculator ratingCalculator = new CumulativeWeightCalculator(tangle, snapshotProvider); TailFinder tailFinder = new TailFinderImpl(tangle); Walker walker = new WalkerAlpha(tailFinder, tangle, new SecureRandom(), configuration); @@ -164,19 +154,18 @@ TipSelector provideTipSelector(Tangle tangle, SnapshotProvider snapshotProvider, Iota provideIota(SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, @Nullable LocalSnapshotManager localSnapshotManager, MilestoneService milestoneService, - LatestMilestoneTracker latestMilestoneTracker, LatestSolidMilestoneTracker latestSolidMilestoneTracker, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, @Nullable TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector, LocalSnapshotsPersistenceProvider localSnapshotsDb, - CacheManager cacheManager) { + CacheManager cacheManager, TransactionSolidifier transactionSolidifier) { return new Iota(configuration, spentAddressesProvider, spentAddressesService, snapshotProvider, snapshotService, - localSnapshotManager, milestoneService, latestMilestoneTracker, latestSolidMilestoneTracker, - seenMilestonesRetriever, ledgerService, transactionPruner, milestoneSolidifier, bundleValidator, tangle, - transactionValidator, transactionRequester, neighborRouter, transactionProcessingPipeline, - tipsRequester, tipsViewModel, tipsSelector, localSnapshotsDb, cacheManager); + localSnapshotManager, milestoneService, seenMilestonesRetriever, ledgerService, + transactionPruner, milestoneSolidifier, bundleValidator, tangle, transactionValidator, + transactionRequester, neighborRouter, transactionProcessingPipeline, tipsRequester, + tipsViewModel, tipsSelector, localSnapshotsDb, cacheManager, transactionSolidifier); } @Singleton @@ -188,11 +177,18 @@ IXI provideIxi(Iota iota) { @Singleton @Provides API provideApi(IXI ixi, TransactionRequester transactionRequester, - SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, - SnapshotProvider snapshotProvider, LedgerService ledgerService, NeighborRouter neighborRouter, TipSelector tipsSelector, - TipsViewModel tipsViewModel, TransactionValidator transactionValidator, - LatestMilestoneTracker latestMilestoneTracker, TransactionProcessingPipeline txPipeline) { - return new API(configuration, ixi, transactionRequester, spentAddressesService, tangle, bundleValidator, snapshotProvider, ledgerService, neighborRouter, tipsSelector, tipsViewModel, transactionValidator, latestMilestoneTracker, txPipeline); + SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, + SnapshotProvider snapshotProvider, LedgerService ledgerService, NeighborRouter neighborRouter, TipSelector tipsSelector, + TipsViewModel tipsViewModel, TransactionValidator transactionValidator, + TransactionProcessingPipeline txPipeline, TransactionSolidifier transactionSolidifier, MilestoneSolidifier milestoneSolidifier) { + return new API(configuration, ixi, transactionRequester, spentAddressesService, tangle, bundleValidator, snapshotProvider, ledgerService, neighborRouter, tipsSelector, tipsViewModel, transactionValidator, milestoneSolidifier, txPipeline, transactionSolidifier); + } + + @Singleton + @Provides + MilestoneSolidifier provideMilestoneSolidifier(TransactionSolidifier transactionSolidifier, Tangle tangle, SnapshotProvider snapshotProvider, LedgerService ledgerService, TransactionRequester transactionRequester, MilestoneService milestoneService){ + return new MilestoneSolidifierImpl(transactionSolidifier,tangle,snapshotProvider, ledgerService, + transactionRequester, milestoneService, configuration); } @Singleton diff --git a/src/main/java/com/iota/iri/TransactionValidator.java b/src/main/java/com/iota/iri/TransactionValidator.java deleted file mode 100644 index ef51b5c203..0000000000 --- a/src/main/java/com/iota/iri/TransactionValidator.java +++ /dev/null @@ -1,465 +0,0 @@ -package com.iota.iri; - -import com.google.common.annotations.VisibleForTesting; -import com.iota.iri.conf.ProtocolConfig; -import com.iota.iri.controllers.TipsViewModel; -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.crypto.Curl; -import com.iota.iri.crypto.Sponge; -import com.iota.iri.crypto.SpongeFactory; -import com.iota.iri.model.Hash; -import com.iota.iri.model.TransactionHash; -import com.iota.iri.network.TransactionRequester; -import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.storage.Tangle; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.iota.iri.controllers.TransactionViewModel.*; - -public class TransactionValidator { - private static final Logger log = LoggerFactory.getLogger(TransactionValidator.class); - private static final int TESTNET_MWM_CAP = 13; - public static final int SOLID_SLEEP_TIME = 500; - - private final Tangle tangle; - private final SnapshotProvider snapshotProvider; - private final TipsViewModel tipsViewModel; - private final TransactionRequester transactionRequester; - private int minWeightMagnitude = 81; - private static final long MAX_TIMESTAMP_FUTURE = 2L * 60L * 60L; - private static final long MAX_TIMESTAMP_FUTURE_MS = MAX_TIMESTAMP_FUTURE * 1_000L; - - - /////////////////////////////////fields for solidification thread////////////////////////////////////// - - private Thread newSolidThread; - - /** - * If true use {@link #newSolidTransactionsOne} while solidifying. Else use {@link #newSolidTransactionsTwo}. - */ - private final AtomicBoolean useFirst = new AtomicBoolean(true); - /** - * Is {@link #newSolidThread} shutting down - */ - private final AtomicBoolean shuttingDown = new AtomicBoolean(false); - /** - * mutex for solidification - */ - private final Object cascadeSync = new Object(); - private final Set newSolidTransactionsOne = new LinkedHashSet<>(); - private final Set newSolidTransactionsTwo = new LinkedHashSet<>(); - - /** - * Constructor for Tangle Validator - * - * @param tangle relays tangle data to and from the persistence layer - * @param snapshotProvider data provider for the snapshots that are relevant for the node - * @param tipsViewModel container that gets updated with the latest tips (transactions with no children) - * @param transactionRequester used to request missing transactions from neighbors - * @param protocolConfig used for checking if we are in testnet and mwm. testnet true if we are in testnet - * mode, this caps {@code mwm} to {@value #TESTNET_MWM_CAP} regardless of parameter input. - * minimum weight magnitude: the minimal number of 9s that ought to appear at the end of the - * transaction hash - */ - TransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, ProtocolConfig protocolConfig) { - this.tangle = tangle; - this.snapshotProvider = snapshotProvider; - this.tipsViewModel = tipsViewModel; - this.transactionRequester = transactionRequester; - this.newSolidThread = new Thread(spawnSolidTransactionsPropagation(), "Solid TX cascader"); - setMwm(protocolConfig.isTestnet(), protocolConfig.getMwm()); - } - - /** - * Does two things: - *
    - *
  1. Sets the minimum weight magnitude (MWM). POW on a transaction is validated by counting a certain - * number of consecutive 9s in the end of the transaction hash. The number of 9s is the MWM.
    2. - *
  2. Starts the transaction solidification thread.
    3. - *
- * - * - * @see #spawnSolidTransactionsPropagation() - */ - public void init() { - newSolidThread.start(); - } - - @VisibleForTesting - void setMwm(boolean testnet, int mwm) { - minWeightMagnitude = mwm; - - //lowest allowed MWM encoded in 46 bytes. - if (!testnet){ - minWeightMagnitude = Math.max(minWeightMagnitude, TESTNET_MWM_CAP); - } - } - - /** - * Shutdown roots to tip solidification thread - * @throws InterruptedException - * @see #spawnSolidTransactionsPropagation() - */ - public void shutdown() throws InterruptedException { - shuttingDown.set(true); - newSolidThread.join(); - } - - /** - * @return the minimal number of trailing 9s that have to be present at the end of the transaction hash - * in order to validate that sufficient proof of work has been done - */ - public int getMinWeightMagnitude() { - return minWeightMagnitude; - } - - /** - * Checks that the timestamp of the transaction is below the last global snapshot time - * or more than {@value #MAX_TIMESTAMP_FUTURE} seconds in the future, and thus invalid. - * - *

- * First the attachment timestamp (set after performing POW) is checked, and if not available - * the regular timestamp is checked. Genesis transaction will always be valid. - *

- * @param transactionViewModel transaction under test - * @return true if timestamp is not in valid bounds and {@code transactionViewModel} is not genesis. - * Else returns false. - */ - private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) { - // ignore invalid timestamps for transactions that were requested by our node while solidifying a milestone - if(transactionRequester.wasTransactionRecentlyRequested(transactionViewModel.getHash())) { - return false; - } - - if (transactionViewModel.getAttachmentTimestamp() == 0) { - return transactionViewModel.getTimestamp() < snapshotProvider.getInitialSnapshot().getTimestamp() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(transactionViewModel.getHash()) - || transactionViewModel.getTimestamp() > (System.currentTimeMillis() / 1000) + MAX_TIMESTAMP_FUTURE; - } - return transactionViewModel.getAttachmentTimestamp() < (snapshotProvider.getInitialSnapshot().getTimestamp() * 1000L) - || transactionViewModel.getAttachmentTimestamp() > System.currentTimeMillis() + MAX_TIMESTAMP_FUTURE_MS; - } - - /** - * Runs the following validation checks on a transaction: - *
    - *
  1. {@link #hasInvalidTimestamp} check.
    2. - *
  2. Check that no value trits are set beyond the usable index, otherwise we will have values larger - * than max supply.
    3. - *
  3. Check that sufficient POW was performed.
    4. - *
  4. In value transactions, we check that the address has 0 set as the last trit. This must be because of the - * conversion between bytes to trits.
    5. - *
- *Exception is thrown upon failure. - * - * @param transactionViewModel transaction that should be validated - * @param minWeightMagnitude the minimal number of trailing 9s at the end of the transaction hash - * @throws StaleTimestampException if timestamp check fails - * @throws IllegalStateException if any of the other checks fail - */ - public void runValidation(TransactionViewModel transactionViewModel, final int minWeightMagnitude) { - transactionViewModel.setMetadata(); - transactionViewModel.setAttachmentData(); - if(hasInvalidTimestamp(transactionViewModel)) { - throw new StaleTimestampException("Invalid transaction timestamp."); - } - for (int i = VALUE_TRINARY_OFFSET + VALUE_USABLE_TRINARY_SIZE; i < VALUE_TRINARY_OFFSET + VALUE_TRINARY_SIZE; i++) { - if (transactionViewModel.trits()[i] != 0) { - throw new IllegalStateException("Invalid transaction value"); - } - } - - int weightMagnitude = transactionViewModel.weightMagnitude; - if(weightMagnitude < minWeightMagnitude) { - throw new IllegalStateException("Invalid weight magnitude"); - } - - if (transactionViewModel.value() != 0 && transactionViewModel.getAddressHash().trits()[Curl.HASH_LENGTH - 1] != 0) { - throw new IllegalStateException("Invalid transaction address"); - } - } - - /** - * Creates a new transaction from {@code trits} and validates it with {@link #runValidation}. - * - * @param trits raw transaction trits - * @param minWeightMagnitude minimal number of trailing 9s in transaction for POW validation - * @return the transaction resulting from the raw trits if valid. - * @throws RuntimeException if validation fails - */ - public TransactionViewModel validateTrits(final byte[] trits, int minWeightMagnitude) { - TransactionViewModel transactionViewModel = new TransactionViewModel(trits, TransactionHash.calculate(trits, 0, trits.length, SpongeFactory.create(SpongeFactory.Mode.CURLP81))); - runValidation(transactionViewModel, minWeightMagnitude); - return transactionViewModel; - } - - /** - * Creates a new transaction from {@code bytes} and validates it with {@link #runValidation}. - * - * @param bytes raw transaction bytes - * @param minWeightMagnitude minimal number of trailing 9s in transaction for POW validation - * @return the transaction resulting from the raw bytes if valid - * @throws RuntimeException if validation fails - */ - public TransactionViewModel validateBytes(final byte[] bytes, int minWeightMagnitude, Sponge curl) { - TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, TransactionHash.calculate(bytes, TRINARY_SIZE, curl)); - runValidation(transactionViewModel, minWeightMagnitude); - return transactionViewModel; - } - - /** - * This method does the same as {@link #checkSolidity(Hash, int)} but defaults to an unlimited amount - * of transactions that are allowed to be traversed. - * - * @param hash hash of the transactions that shall get checked - * @return true if the transaction is solid and false otherwise - * @throws Exception if anything goes wrong while trying to solidify the transaction - */ - public boolean checkSolidity(Hash hash) throws Exception { - return checkSolidity(hash, Integer.MAX_VALUE); - } - - /** - * This method checks transactions for solidity and marks them accordingly if they are found to be solid. - * - * It iterates through all approved transactions until it finds one that is missing in the database or until it - * reached solid transactions on all traversed subtangles. In case of a missing transactions it issues a transaction - * request and returns false. If no missing transaction is found, it marks the processed transactions as solid in - * the database and returns true. - * - * Since this operation can potentially take a long time to terminate if it would have to traverse big parts of the - * tangle, it is possible to limit the amount of transactions that are allowed to be processed, while looking for - * unsolid / missing approvees. This can be useful when trying to "interrupt" the solidification of one transaction - * (if it takes too many steps) to give another one the chance to be solidified instead (i.e. prevent blocks in the - * solidification threads). - * - * @param hash hash of the transactions that shall get checked - * @param maxProcessedTransactions the maximum amount of transactions that are allowed to be traversed - * @return true if the transaction is solid and false otherwise - * @throws Exception if anything goes wrong while trying to solidify the transaction - */ - public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception { - if(fromHash(tangle, hash).isSolid()) { - return true; - } - LinkedHashSet analyzedHashes = new LinkedHashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); - if(maxProcessedTransactions != Integer.MAX_VALUE) { - maxProcessedTransactions += analyzedHashes.size(); - } - boolean solid = true; - final Queue nonAnalyzedTransactions = new LinkedList<>(Collections.singleton(hash)); - Hash hashPointer; - while ((hashPointer = nonAnalyzedTransactions.poll()) != null) { - if (!analyzedHashes.add(hashPointer)) { - continue; - } - - if (analyzedHashes.size() >= maxProcessedTransactions) { - return false; - } - - TransactionViewModel transaction = fromHash(tangle, hashPointer); - if (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { - if (transaction.getType() == PREFILLED_SLOT) { - solid = false; - - if (!transactionRequester.isTransactionRequested(hashPointer)) { - transactionRequester.requestTransaction(hashPointer); - continue; - } - } else { - nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); - nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); - } - } - } - if (solid) { - updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), analyzedHashes); - } - analyzedHashes.clear(); - return solid; - } - - public void addSolidTransaction(Hash hash) { - synchronized (cascadeSync) { - if (useFirst.get()) { - newSolidTransactionsOne.add(hash); - } else { - newSolidTransactionsTwo.add(hash); - } - } - } - - /** - * Creates a runnable that runs {@link #propagateSolidTransactions()} in a loop every {@value #SOLID_SLEEP_TIME} ms - * @return runnable that is not started - */ - private Runnable spawnSolidTransactionsPropagation() { - return () -> { - while(!shuttingDown.get()) { - propagateSolidTransactions(); - try { - Thread.sleep(SOLID_SLEEP_TIME); - } catch (InterruptedException e) { - // Ignoring InterruptedException. Do not use Thread.currentThread().interrupt() here. - log.error("Thread was interrupted: ", e); - } - } - }; - } - - /** - * Iterates over all currently known solid transactions. For each solid transaction, we find - * its children (approvers) and try to quickly solidify them with {@link #quietQuickSetSolid}. - * If we manage to solidify the transactions, we add them to the solidification queue for a traversal by a later run. - */ - @VisibleForTesting - void propagateSolidTransactions() { - Set newSolidHashes = new HashSet<>(); - useFirst.set(!useFirst.get()); - //synchronized to make sure no one is changing the newSolidTransactions collections during addAll - synchronized (cascadeSync) { - //We are using a collection that doesn't get updated by other threads - if (useFirst.get()) { - newSolidHashes.addAll(newSolidTransactionsTwo); - newSolidTransactionsTwo.clear(); - } else { - newSolidHashes.addAll(newSolidTransactionsOne); - newSolidTransactionsOne.clear(); - } - } - Iterator cascadeIterator = newSolidHashes.iterator(); - while(cascadeIterator.hasNext() && !shuttingDown.get()) { - try { - Hash hash = cascadeIterator.next(); - TransactionViewModel transaction = fromHash(tangle, hash); - Set approvers = transaction.getApprovers(tangle).getHashes(); - for(Hash h: approvers) { - TransactionViewModel tx = fromHash(tangle, h); - if(quietQuickSetSolid(tx)) { - tx.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); - tipsViewModel.setSolid(h); - addSolidTransaction(h); - } - } - } catch (Exception e) { - log.error("Error while propagating solidity upwards", e); - } - } - } - - - /** - * Updates a transaction after it was stored in the tangle. Tells the node to not request the transaction anymore, - * to update the live tips accordingly, and attempts to quickly solidify the transaction. - * - *

- * Performs the following operations: - * - *

    - *
  1. Removes {@code transactionViewModel}'s hash from the the request queue since we already found it.
    2. - *
  2. If {@code transactionViewModel} has no children (approvers), we add it to the node's active tip list.
    3. - *
  3. Removes {@code transactionViewModel}'s parents (branch & trunk) from the node's tip list - * (if they're present there).
    4. - *
  4. Attempts to quickly solidify {@code transactionViewModel} by checking whether its direct parents - * are solid. If solid we add it to the queue transaction solidification thread to help it propagate the - * solidification to the approving child transactions.
    5. - *
  5. Requests missing direct parent (trunk & branch) transactions that are needed to solidify - * {@code transactionViewModel}.
    6. - *
- * @param transactionViewModel received transaction that is being updated - * @throws Exception if an error occurred while trying to solidify - * @see TipsViewModel - */ - //Not part of the validation process. This should be moved to a component in charge of - //what transaction we gossip. - public void updateStatus(TransactionViewModel transactionViewModel) throws Exception { - transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); - if(transactionViewModel.getApprovers(tangle).size() == 0) { - tipsViewModel.addTipHash(transactionViewModel.getHash()); - } - tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); - tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); - - if(quickSetSolid(transactionViewModel)) { - transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); - tipsViewModel.setSolid(transactionViewModel.getHash()); - addSolidTransaction(transactionViewModel.getHash()); - } - } - - /** - * Perform a {@link #quickSetSolid} while capturing and logging errors - * @param transactionViewModel transaction we try to solidify. - * @return true if we managed to solidify, else false. - */ - private boolean quietQuickSetSolid(TransactionViewModel transactionViewModel) { - try { - return quickSetSolid(transactionViewModel); - } catch (Exception e) { - log.error(e.getMessage(), e); - return false; - } - } - - /** - * Tries to solidify the transactions quickly by performing {@link #checkApproovee} on both parents (trunk and - * branch). If the parents are solid, mark the transactions as solid. - * @param transactionViewModel transaction to solidify - * @return true if we made the transaction solid, else false. - * @throws Exception - */ - private boolean quickSetSolid(final TransactionViewModel transactionViewModel) throws Exception { - if(!transactionViewModel.isSolid()) { - boolean solid = true; - if (!checkApproovee(transactionViewModel.getTrunkTransaction(tangle))) { - solid = false; - } - if (!checkApproovee(transactionViewModel.getBranchTransaction(tangle))) { - solid = false; - } - if(solid) { - transactionViewModel.updateSolid(true); - transactionViewModel.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); - return true; - } - } - return false; - } - - /** - * If the the {@code approvee} is missing, request it from a neighbor. - * @param approovee transaction we check. - * @return true if {@code approvee} is solid. - * @throws Exception if we encounter an error while requesting a transaction - */ - private boolean checkApproovee(TransactionViewModel approovee) throws Exception { - if(snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(approovee.getHash())) { - return true; - } - if(approovee.getType() == PREFILLED_SLOT) { - // don't solidify from the bottom until cuckoo filters can identify where we deleted -> otherwise we will - // continue requesting old transactions forever - //transactionRequester.requestTransaction(approovee.getHash(), false); - return false; - } - return approovee.isSolid(); - } - - @VisibleForTesting - boolean isNewSolidTxSetsEmpty () { - return newSolidTransactionsOne.isEmpty() && newSolidTransactionsTwo.isEmpty(); - } - - /** - * Thrown if transaction fails {@link #hasInvalidTimestamp} check. - */ - public static class StaleTimestampException extends RuntimeException { - StaleTimestampException (String message) { - super(message); - } - } -} diff --git a/src/main/java/com/iota/iri/conf/SolidificationConfig.java b/src/main/java/com/iota/iri/conf/SolidificationConfig.java index 495b11cd70..2492cf98fc 100644 --- a/src/main/java/com/iota/iri/conf/SolidificationConfig.java +++ b/src/main/java/com/iota/iri/conf/SolidificationConfig.java @@ -1,5 +1,7 @@ package com.iota.iri.conf; +import com.iota.iri.model.Hash; + /** * * Configurations that should be used for the solidification processes. @@ -13,6 +15,8 @@ public interface SolidificationConfig extends Config { */ boolean isPrintSyncProgressEnabled(); + Hash getCoordinator(); + /** * Field descriptions */ diff --git a/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java b/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java index 244c660a03..a46c25fefa 100644 --- a/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java +++ b/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java @@ -14,7 +14,7 @@ /** * Acts as a controller interface for a {@link Milestone} hash object. This controller is used by the - * {@link com.iota.iri.MilestoneTracker} to manipulate a {@link Milestone} object. + * {@link com.iota.iri.service.milestone.MilestoneSolidifier} to manipulate a {@link Milestone} object. */ public class MilestoneViewModel { private final Milestone milestone; diff --git a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java index 89e78407ed..3ce1c0056e 100644 --- a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java +++ b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java @@ -5,6 +5,7 @@ import com.iota.iri.model.*; import com.iota.iri.model.persistables.*; import com.iota.iri.service.snapshot.Snapshot; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.Tangle; @@ -778,33 +779,11 @@ public void setMetadata() { : TransactionViewModel.FILLED_SLOT; } - /** - * Update solid transactions - * @param tangle Tangle - * @param initialSnapshot Initial snapshot - * @param analyzedHashes analyzed hashes - * @throws Exception Exception - */ - public static void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, - final Set analyzedHashes) throws Exception { - Object[] hashes = analyzedHashes.toArray(); - TransactionViewModel transactionViewModel; - for (int i = hashes.length - 1; i >= 0; i--) { - transactionViewModel = TransactionViewModel.fromHash(tangle, (Hash) hashes[i]); - - transactionViewModel.updateHeights(tangle, initialSnapshot); - - if (!transactionViewModel.isSolid()) { - transactionViewModel.updateSolid(true); - transactionViewModel.update(tangle, initialSnapshot, "solid|height"); - } - } - } /** * Updates the {@link Transaction#solid} value of the referenced {@link Transaction} object. * - * Used by the {@link com.iota.iri.TransactionValidator} to quickly set the solidity of a {@link Transaction} set. + * Used by the {@link TransactionValidator} to quickly set the solidity of a {@link Transaction} set. * * @param solid The solidity of the transaction in the database * @return True if the {@link Transaction#solid} has been updated, False if not. @@ -847,7 +826,7 @@ public void setSnapshot(Tangle tangle, Snapshot initialSnapshot, final int index /** * This method sets the {@link Transaction#milestone} flag. * - * It gets automatically called by the {@link com.iota.iri.service.milestone.LatestMilestoneTracker} and marks + * It gets automatically called by the {@link com.iota.iri.service.milestone.MilestoneSolidifier} and marks * transactions that represent a milestone accordingly. It first checks if the {@link Transaction#milestone} flag * has changed and if so, it issues a database update. * diff --git a/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java b/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java index 032715736d..ee247e1bb7 100644 --- a/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java @@ -3,13 +3,15 @@ import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.IotaConfig; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.network.impl.TipsRequesterImpl; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.network.pipeline.TransactionProcessingPipelineImpl; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; @@ -37,18 +39,21 @@ TransactionRequester provideTransactionRequester(Tangle tangle, SnapshotProvider @Singleton @Provides - TipsRequester provideTipsRequester(NeighborRouter neighborRouter, Tangle tangle, LatestMilestoneTracker latestMilestoneTracker, TransactionRequester txRequester) { - return new TipsRequesterImpl(neighborRouter, tangle, latestMilestoneTracker, txRequester); + TipsRequester provideTipsRequester(NeighborRouter neighborRouter, Tangle tangle, + MilestoneSolidifier milestoneSolidifier, TransactionRequester txRequester) { + return new TipsRequesterImpl(neighborRouter, tangle, milestoneSolidifier, txRequester); } @Singleton @Provides TransactionProcessingPipeline provideTransactionProcessingPipeline(NeighborRouter neighborRouter, TransactionValidator txValidator, Tangle tangle, SnapshotProvider snapshotProvider, - TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, - TransactionRequester transactionRequester) { + TipsViewModel tipsViewModel, TransactionRequester transactionRequester, + TransactionSolidifier transactionSolidifier, MilestoneService milestoneService, + MilestoneSolidifier milestoneSolidifier) { return new TransactionProcessingPipelineImpl(neighborRouter, configuration, txValidator, tangle, - snapshotProvider, tipsViewModel, latestMilestoneTracker, transactionRequester); + snapshotProvider, tipsViewModel, transactionRequester, transactionSolidifier, + milestoneService, milestoneSolidifier); } @Singleton diff --git a/src/main/java/com/iota/iri/network/impl/TipsRequesterImpl.java b/src/main/java/com/iota/iri/network/impl/TipsRequesterImpl.java index 58d0405e15..19697ffa5e 100644 --- a/src/main/java/com/iota/iri/network/impl/TipsRequesterImpl.java +++ b/src/main/java/com/iota/iri/network/impl/TipsRequesterImpl.java @@ -6,7 +6,7 @@ import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.Neighbor; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.storage.Tangle; import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; import com.iota.iri.utils.thread.SilentScheduledExecutorService; @@ -29,8 +29,8 @@ public class TipsRequesterImpl implements TipsRequester { private final NeighborRouter neighborRouter; private final Tangle tangle; private final TransactionRequester txRequester; - private final LatestMilestoneTracker latestMilestoneTracker; + private MilestoneSolidifier milestoneSolidifier; private long lastIterationTime = 0; /** @@ -38,16 +38,16 @@ public class TipsRequesterImpl implements TipsRequester { * * @param neighborRouter the {@link NeighborRouter} to use * @param tangle the {@link Tangle} database to load the latest milestone from - * @param latestMilestoneTracker the {@link LatestMilestoneTracker} to gets the latest milestone hash from + * @param milestoneSolidifier the {@link MilestoneSolidifier} to gets the latest milestone hash from * @param txRequester the {@link TransactionRequester} to get the currently number of requested * transactions from */ - public TipsRequesterImpl(NeighborRouter neighborRouter, Tangle tangle, LatestMilestoneTracker latestMilestoneTracker, + public TipsRequesterImpl(NeighborRouter neighborRouter, Tangle tangle, MilestoneSolidifier milestoneSolidifier, TransactionRequester txRequester) { this.neighborRouter = neighborRouter; this.tangle = tangle; - this.latestMilestoneTracker = latestMilestoneTracker; this.txRequester = txRequester; + this.milestoneSolidifier = milestoneSolidifier; } /** @@ -64,7 +64,7 @@ public void start() { public void requestTips() { try { final TransactionViewModel msTVM = TransactionViewModel.fromHash(tangle, - latestMilestoneTracker.getLatestMilestoneHash()); + milestoneSolidifier.getLatestMilestoneHash()); if (msTVM.getBytes().length > 0) { for (Neighbor neighbor : neighborRouter.getConnectedNeighbors().values()) { diff --git a/src/main/java/com/iota/iri/network/pipeline/MilestonePayload.java b/src/main/java/com/iota/iri/network/pipeline/MilestonePayload.java new file mode 100644 index 0000000000..7c63ada07d --- /dev/null +++ b/src/main/java/com/iota/iri/network/pipeline/MilestonePayload.java @@ -0,0 +1,56 @@ +package com.iota.iri.network.pipeline; + +import com.iota.iri.model.Hash; +import com.iota.iri.network.neighbor.Neighbor; + +/** + * A payload object for processing milestone candidates + */ +public class MilestonePayload extends Payload { + /** + * Origin neighbor for transaction + */ + private Neighbor originNeighbor; + + /** + * {@link Hash} of potential milestone object + */ + private Hash milestoneHash; + + /** + * Index of potential milestone object + */ + private int milestoneIndex; + + /** + * Constructor for a {@link MilestonePayload} object that will be processed by the {@link MilestoneStage}. + * + * @param originNeighbor Neighbor that milestone candidate originated from + * @param milestoneHash {@link Hash} of the milestone candidate + * @param milestoneIndex Index of the milestone candidate + */ + public MilestonePayload(Neighbor originNeighbor, Hash milestoneHash, int milestoneIndex){ + this.originNeighbor = originNeighbor; + this.milestoneHash = milestoneHash; + this.milestoneIndex = milestoneIndex; + } + + @Override + public Neighbor getOriginNeighbor() { + return originNeighbor; + } + + /** + * @return {@link #milestoneHash} + */ + public Hash getMilestoneHash(){ + return this.milestoneHash; + } + + /** + * @return {@link #milestoneIndex} + */ + public int getMilestoneIndex(){ + return this.milestoneIndex; + } +} diff --git a/src/main/java/com/iota/iri/network/pipeline/MilestoneStage.java b/src/main/java/com/iota/iri/network/pipeline/MilestoneStage.java new file mode 100644 index 0000000000..a0559cf525 --- /dev/null +++ b/src/main/java/com/iota/iri/network/pipeline/MilestoneStage.java @@ -0,0 +1,107 @@ +package com.iota.iri.network.pipeline; + +import com.iota.iri.controllers.MilestoneViewModel; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.storage.Tangle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Stage for processing {@link MilestonePayload} objects + */ +public class MilestoneStage implements Stage { + + private static final Logger log = LoggerFactory.getLogger(MilestoneStage.class); + + + private MilestoneSolidifier milestoneSolidifier; + private Tangle tangle; + private SnapshotProvider snapshotProvider; + private MilestoneService milestoneService; + private TransactionSolidifier transactionSolidifier; + + /** + * Constructor for {@link MilestoneStage}. This stage will process {@link MilestonePayload} candidate objects using + * the {@link MilestoneSolidifier} to determine the latest milestone, the latest solid milestone, and place unsolid + * milestone candidates into a queue for solidification. The stage will always try to solidify the oldest milestone + * candidate in the queue. + * + * @param tangle The tangle reference + * @param milestoneSolidifier Solidification service for processing milestone objects + * @param snapshotProvider Snapshot provider service for latest snapshot references + * @param milestoneService A service for validating milestone objects + * @param transactionSolidifier A service for solidifying transactions + */ + public MilestoneStage(Tangle tangle, MilestoneSolidifier milestoneSolidifier, + SnapshotProvider snapshotProvider, MilestoneService milestoneService, + TransactionSolidifier transactionSolidifier){ + this.milestoneSolidifier = milestoneSolidifier; + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + this.milestoneService = milestoneService; + this.transactionSolidifier = transactionSolidifier; + } + + /** + * Process {@link MilestonePayload} objects. While processing the {@link MilestoneStage} will determine the latest + * milestone and log it. If the milestone object passes a validity check, it is added to the seenMilestones queue in + * the {@link MilestoneSolidifier}, whereas if it is invalid, the transaction is ignored. The milestone is then + * checked for solidity using the {@link TransactionSolidifier}. If the transaction is solid it is passed forward to + * the {@link TransactionSolidifier} propagation queue. + * + * @param ctx the context to process + * @return Either an abort or solidify stage ctx + */ + @Override + public ProcessingContext process(ProcessingContext ctx){ + try { + MilestonePayload payload = (MilestonePayload) ctx.getPayload(); + + //If the milestone index is below the latest snapshot initial index, then abort the process + if(payload.getMilestoneIndex() < snapshotProvider.getLatestSnapshot().getInitialIndex()){ + return abort(ctx); + } + + TransactionViewModel milestone = TransactionViewModel.fromHash(tangle, payload.getMilestoneHash()); + int newMilestoneIndex = payload.getMilestoneIndex(); + boolean isFirstInBundle = (milestone.getCurrentIndex() == 0); + + // Log new milestones + int latestMilestoneIndex = milestoneSolidifier.getLatestMilestoneIndex(); + if (newMilestoneIndex > latestMilestoneIndex) { + milestoneSolidifier.logNewMilestone(latestMilestoneIndex, newMilestoneIndex, milestone.getHash()); + } + + // Add unsolid milestones to the milestone solidifier, or add solid milestones to the propagation queue + if (!transactionSolidifier.addMilestoneToSolidificationQueue(milestone.getHash(), 50000) && + isFirstInBundle) { + milestoneSolidifier.add(milestone.getHash(), payload.getMilestoneIndex()); + } else { + transactionSolidifier.addToPropagationQueue(milestone.getHash()); + } + + return solidify(ctx, payload, milestone); + }catch (Exception e){ + log.error("Error processing milestone: ", e); + return abort(ctx); + } + } + + + private ProcessingContext solidify(ProcessingContext ctx, Payload payload, TransactionViewModel tvm){ + SolidifyPayload solidifyPayload = new SolidifyPayload(payload.getOriginNeighbor(), tvm); + ctx.setNextStage(TransactionProcessingPipeline.Stage.SOLIDIFY); + ctx.setPayload(solidifyPayload); + return ctx; + } + + private ProcessingContext abort(ProcessingContext ctx){ + ctx.setNextStage(TransactionProcessingPipeline.Stage.ABORT); + return ctx; + } +} diff --git a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java index e8c5e99711..7cd6f8561f 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java @@ -1,6 +1,9 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.model.Hash; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.Neighbor; @@ -19,27 +22,32 @@ public class ReceivedStage implements Stage { private Tangle tangle; private TransactionRequester transactionRequester; - private TransactionValidator txValidator; + private TransactionSolidifier txSolidifier; private SnapshotProvider snapshotProvider; + private MilestoneService milestoneService; + private Hash cooAddress; /** * Creates a new {@link ReceivedStage}. * * @param tangle The {@link Tangle} database used to store/update the transaction - * @param txValidator The {@link TransactionValidator} used to store/update the transaction + * @param txSolidifier The {@link TransactionValidator} used to store/update the transaction * @param snapshotProvider The {@link SnapshotProvider} used to store/update the transaction */ - public ReceivedStage(Tangle tangle, TransactionValidator txValidator, SnapshotProvider snapshotProvider, - TransactionRequester transactionRequester) { - this.txValidator = txValidator; + public ReceivedStage(Tangle tangle, TransactionSolidifier txSolidifier, SnapshotProvider snapshotProvider, + TransactionRequester transactionRequester, MilestoneService milestoneService, + Hash cooAddress) { + this.txSolidifier = txSolidifier; this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.transactionRequester = transactionRequester; + this.milestoneService = milestoneService; + this.cooAddress = cooAddress; } /** * Stores the given transaction in the database, updates it status - * ({@link TransactionValidator#updateStatus(TransactionViewModel)}) and updates the sender. + * ({@link TransactionSolidifier#updateStatus(TransactionViewModel)}) and updates the sender. * * @param ctx the received stage {@link ProcessingContext} * @return a {@link ProcessingContext} which redirects to the {@link BroadcastStage} @@ -65,7 +73,7 @@ public ProcessingContext process(ProcessingContext ctx) { if (stored) { tvm.setArrivalTime(System.currentTimeMillis()); try { - txValidator.updateStatus(tvm); + txSolidifier.updateStatus(tvm); // free up the recently requested transaction set if(transactionRequester.removeRecentlyRequestedTransaction(tvm.getHash())){ @@ -90,9 +98,22 @@ public ProcessingContext process(ProcessingContext ctx) { transactionRequester.removeRecentlyRequestedTransaction(tvm.getHash()); } + try{ + if(tvm.getAddressHash().equals(cooAddress)) { + int milestoneIndex = milestoneService.getMilestoneIndex(tvm); + MilestonePayload milestonePayload = new MilestonePayload(payload.getOriginNeighbor(), + tvm.getHash(), milestoneIndex); + ctx.setNextStage(TransactionProcessingPipeline.Stage.MILESTONE); + ctx.setPayload(milestonePayload); + return ctx; + } + + } catch(Exception e){ + log.error("Error checking apparent milestone transaction", e); + } // broadcast the newly saved tx to the other neighbors - ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); - ctx.setPayload(new BroadcastPayload(originNeighbor, tvm)); + ctx.setNextStage(TransactionProcessingPipeline.Stage.SOLIDIFY); + ctx.setPayload(new SolidifyPayload(originNeighbor, tvm)); return ctx; } } diff --git a/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java b/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java index 28be79ac3a..2aa5ef2024 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java @@ -10,9 +10,9 @@ import com.iota.iri.network.TransactionCacheDigester; import com.iota.iri.network.neighbor.Neighbor; import com.iota.iri.network.protocol.Protocol; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; +import com.iota.iri.service.milestone.MilestoneSolidifier; import java.security.SecureRandom; @@ -33,7 +33,7 @@ public class ReplyStage implements Stage { private Tangle tangle; private NodeConfig config; private TipsViewModel tipsViewModel; - private LatestMilestoneTracker latestMilestoneTracker; + private MilestoneSolidifier milestoneSolidifier; private SnapshotProvider snapshotProvider; private FIFOCache recentlySeenBytesCache; private SecureRandom rnd = new SecureRandom(); @@ -45,20 +45,20 @@ public class ReplyStage implements Stage { * @param config the {@link NodeConfig} * @param tangle the {@link Tangle} database to load the request transaction from * @param tipsViewModel the {@link TipsViewModel} to load the random tips from - * @param latestMilestoneTracker the {@link LatestMilestoneTracker} to load the latest milestone from + * @param milestoneSolidifier the {@link MilestoneSolidifier} to load the latest milestone from * @param snapshotProvider the {@link SnapshotProvider} to check the latest solid milestone from * @param recentlySeenBytesCache the {@link FIFOCache} to use to cache the replied transaction * @param rnd the {@link SecureRandom} used to get random values to randomize chances for not * replying at all or not requesting a not stored requested transaction from neighbors */ public ReplyStage(NeighborRouter neighborRouter, NodeConfig config, Tangle tangle, TipsViewModel tipsViewModel, - LatestMilestoneTracker latestMilestoneTracker, SnapshotProvider snapshotProvider, - FIFOCache recentlySeenBytesCache, SecureRandom rnd) { + MilestoneSolidifier milestoneSolidifier, SnapshotProvider snapshotProvider, + FIFOCache recentlySeenBytesCache, SecureRandom rnd) { this.neighborRouter = neighborRouter; this.config = config; this.tangle = tangle; this.tipsViewModel = tipsViewModel; - this.latestMilestoneTracker = latestMilestoneTracker; + this.milestoneSolidifier = milestoneSolidifier; this.snapshotProvider = snapshotProvider; this.recentlySeenBytesCache = recentlySeenBytesCache; this.rnd = rnd; @@ -71,18 +71,18 @@ public ReplyStage(NeighborRouter neighborRouter, NodeConfig config, Tangle tangl * @param config the {@link NodeConfig} * @param tangle the {@link Tangle} database to load the request transaction from * @param tipsViewModel the {@link TipsViewModel} to load the random tips from - * @param latestMilestoneTracker the {@link LatestMilestoneTracker} to load the latest milestone from + * @param milestoneSolidifier the {@link MilestoneSolidifier} to load the latest milestone from * @param snapshotProvider the {@link SnapshotProvider} to check the latest solid milestone from * @param recentlySeenBytesCache the {@link FIFOCache} to use to cache the replied transaction */ public ReplyStage(NeighborRouter neighborRouter, NodeConfig config, Tangle tangle, TipsViewModel tipsViewModel, - LatestMilestoneTracker latestMilestoneTracker, SnapshotProvider snapshotProvider, - FIFOCache recentlySeenBytesCache) { + MilestoneSolidifier milestoneSolidifier, SnapshotProvider snapshotProvider, + FIFOCache recentlySeenBytesCache) { this.neighborRouter = neighborRouter; this.config = config; this.tangle = tangle; this.tipsViewModel = tipsViewModel; - this.latestMilestoneTracker = latestMilestoneTracker; + this.milestoneSolidifier = milestoneSolidifier; this.snapshotProvider = snapshotProvider; this.recentlySeenBytesCache = recentlySeenBytesCache; } @@ -106,7 +106,7 @@ public ProcessingContext process(ProcessingContext ctx) { try { // don't reply to random tip requests if we are synchronized with a max delta of one // to the newest milestone - if (snapshotProvider.getLatestSnapshot().getIndex() >= latestMilestoneTracker.getLatestMilestoneIndex() + if (snapshotProvider.getLatestSnapshot().getIndex() >= milestoneSolidifier.getLatestMilestoneIndex() - 1) { ctx.setNextStage(TransactionProcessingPipeline.Stage.FINISH); return ctx; @@ -151,7 +151,7 @@ public ProcessingContext process(ProcessingContext ctx) { } private Hash getRandomTipPointer() { - Hash tip = rnd.nextDouble() < config.getpSendMilestone() ? latestMilestoneTracker.getLatestMilestoneHash() + Hash tip = rnd.nextDouble() < config.getpSendMilestone() ? milestoneSolidifier.getLatestMilestoneHash() : tipsViewModel.getRandomSolidTipHash(); return tip == null ? Hash.NULL_HASH : tip; } diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java new file mode 100644 index 0000000000..718a2c88bb --- /dev/null +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java @@ -0,0 +1,39 @@ +package com.iota.iri.network.pipeline; + +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.network.neighbor.Neighbor; + +/** + * Defines a payload which gets submitted to the {@link SolidifyStage}. + */ +public class SolidifyPayload extends Payload { + private Neighbor originNeighbor; + private TransactionViewModel tvm; + + /** + * Constructor for solidification payload. + * + * @param originNeighbor The originating point of a received transaction + * @param tvm The transaction that needs to be solidified + */ + public SolidifyPayload(Neighbor originNeighbor, TransactionViewModel tvm){ + this.originNeighbor = originNeighbor; + this.tvm = tvm; + } + + /** + * {@inheritDoc} + */ + @Override + public Neighbor getOriginNeighbor(){ + return originNeighbor; + } + + /** + * Fetches the transaction from the payload. + * @return The transaction stored in the payload. + */ + public TransactionViewModel getTransaction(){ + return tvm; + } +} diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java new file mode 100644 index 0000000000..49f285a509 --- /dev/null +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java @@ -0,0 +1,95 @@ +package com.iota.iri.network.pipeline; + +import com.google.common.annotations.VisibleForTesting; +import com.iota.iri.controllers.TipsViewModel; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.storage.Tangle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.iota.iri.controllers.TransactionViewModel.fromHash; + +/** + * The {@link SolidifyStage} is used to process newly received transaction for solidity. Once a transaction has been + * passed from the {@link ReceivedStage} it will be placed into this stage to have the {@link TransactionSolidifier} + * check the solidity of the transaction. If the transaction is found to be solid, it will be passed forward to the + * {@link BroadcastStage}. If it is found to be unsolid, it is put through the solidity check so that missing reference + * transactions get requested. If the transaction is unsolid, a random solid tip is broadcast instead to keep the + * requests transmitting to neighbors. + */ +public class SolidifyStage implements Stage { + private static final Logger log = LoggerFactory.getLogger(SolidifyStage.class); + + private TransactionSolidifier txSolidifier; + private TipsViewModel tipsViewModel; + private Tangle tangle; + private TransactionViewModel tip; + + /** + * Constructor for the {@link SolidifyStage}. + * + * @param txSolidifier Transaction validator implementation for determining the validity of a transaction + * @param tipsViewModel Used for broadcasting random solid tips if the subject transaction is unsolid + * @param tangle A reference to the nodes DB + */ + public SolidifyStage(TransactionSolidifier txSolidifier, TipsViewModel tipsViewModel, Tangle tangle){ + this.txSolidifier = txSolidifier; + this.tipsViewModel = tipsViewModel; + this.tangle = tangle; + } + + /** + * Processes the payload of the {@link ProcessingContext} as a {@link SolidifyPayload}. First the transaction will + * be checked for solidity and validity. If the transaction is already solid or can be set solid quickly by the + * transaction validator, the transaction is passed to the {@link BroadcastStage}. If not, a random solid tip is + * pulled form the {@link TipsViewModel} to be broadcast instead. + * + * @param ctx The context to process + * @return The output context, in most cases a {@link BroadcastPayload}. + */ + @Override + public ProcessingContext process(ProcessingContext ctx){ + try { + SolidifyPayload payload = (SolidifyPayload) ctx.getPayload(); + TransactionViewModel tvm = payload.getTransaction(); + + if (tvm.isSolid() || txSolidifier.quickSetSolid(tvm)) { + tip = tvm; + } + + return broadcastTip(ctx, payload); + }catch (Exception e){ + log.error("Failed to process transaction for solidification", e); + ctx.setNextStage(TransactionProcessingPipeline.Stage.ABORT); + return ctx; + } + + } + + private ProcessingContext broadcastTip(ProcessingContext ctx, SolidifyPayload payload) throws Exception{ + if(tip == null) { + Hash tipHash = tipsViewModel.getRandomSolidTipHash(); + + if (tipHash == null) { + ctx.setNextStage(TransactionProcessingPipeline.Stage.FINISH); + return ctx; + } + + tip = fromHash(tangle, tipHash); + } + + ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); + ctx.setPayload(new BroadcastPayload(payload.getOriginNeighbor(), tip)); + + tip = null; + return ctx; + } + + @VisibleForTesting + void injectTip(TransactionViewModel tvm) throws Exception { + tip = tvm; + tip.updateSolid(true); + } +} diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java index 1f19e4248c..3b95a31cef 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java @@ -14,7 +14,7 @@ public interface TransactionProcessingPipeline { * Defines the different stages of the {@link TransactionProcessingPipelineImpl}. */ enum Stage { - PRE_PROCESS, HASHING, VALIDATION, REPLY, RECEIVED, BROADCAST, MULTIPLE, ABORT, FINISH, + PRE_PROCESS, HASHING, VALIDATION, REPLY, RECEIVED, BROADCAST, MULTIPLE, ABORT, FINISH, SOLIDIFY, MILESTONE } /** @@ -50,6 +50,13 @@ enum Stage { */ BlockingQueue getValidationStageQueue(); + /** + * Gets the milestone stage queue. + * + * @return the milestone stage queue + */ + BlockingQueue getMilestoneStageQueue(); + /** * Submits the given data from the given neighbor into the pre processing stage of the pipeline. * @@ -111,4 +118,19 @@ enum Stage { * @param hashingStage the {@link HashingStage} to use */ void setHashingStage(HashingStage hashingStage); + + /** + * Sets the solidify stage. This method should only be used for injecting mocked objects. + * + * @param solidifyStage the {@link SolidifyStage} to use + */ + void setSolidifyStage(SolidifyStage solidifyStage); + + /** + * Sets the milestone stage. This method should only be used for injecting mocked objects. + * + * @param milestoneStage the {@link MilestoneStage} to use + */ + void setMilestoneStage(MilestoneStage milestoneStage); + } diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index 0af71b8420..2de89774de 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -1,8 +1,12 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.NodeConfig; import com.iota.iri.controllers.TipsViewModel; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.crypto.batched.BatchedHasher; import com.iota.iri.crypto.batched.BatchedHasherFactory; import com.iota.iri.crypto.batched.HashRequest; @@ -13,13 +17,15 @@ import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.TransactionCacheDigester; import com.iota.iri.network.neighbor.Neighbor; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; import com.iota.iri.utils.Converter; import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; @@ -69,12 +75,18 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP private BroadcastStage broadcastStage; private BatchedHasher batchedHasher; private HashingStage hashingStage; + private SolidifyStage solidifyStage; + private MilestoneStage milestoneStage; + private TransactionSolidifier txSolidifier; + private MilestoneSolidifier milestoneSolidifier; private BlockingQueue preProcessStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue validationStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue receivedStageQueue = new ArrayBlockingQueue<>(100); - private BlockingQueue broadcastStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue replyStageQueue = new ArrayBlockingQueue<>(100); + private BlockingQueue broadcastStageQueue = new ArrayBlockingQueue<>(100); + private BlockingQueue solidifyStageQueue = new ArrayBlockingQueue<>(100); + private BlockingQueue milestoneStageQueue = new ArrayBlockingQueue<>(100); /** * Creates a {@link TransactionProcessingPipeline}. @@ -85,22 +97,29 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP * @param tangle The {@link Tangle} database to use to store and load transactions. * @param snapshotProvider The {@link SnapshotProvider} to use to store transactions with. * @param tipsViewModel The {@link TipsViewModel} to load tips from in the reply stage - * @param latestMilestoneTracker The {@link LatestMilestoneTracker} to load the latest milestone hash from in the + * @param milestoneSolidifier The {@link MilestoneSolidifier} to load the latest milestone hash from in the * reply stage */ public TransactionProcessingPipelineImpl(NeighborRouter neighborRouter, NodeConfig config, TransactionValidator txValidator, Tangle tangle, SnapshotProvider snapshotProvider, - TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, - TransactionRequester transactionRequester) { + TipsViewModel tipsViewModel, TransactionRequester transactionRequester, + TransactionSolidifier txSolidifier, MilestoneService milestoneService, + MilestoneSolidifier milestoneSolidifier) { FIFOCache recentlySeenBytesCache = new FIFOCache<>(config.getCacheSizeBytes()); this.preProcessStage = new PreProcessStage(recentlySeenBytesCache); - this.replyStage = new ReplyStage(neighborRouter, config, tangle, tipsViewModel, latestMilestoneTracker, + this.replyStage = new ReplyStage(neighborRouter, config, tangle, tipsViewModel, milestoneSolidifier, snapshotProvider, recentlySeenBytesCache); this.broadcastStage = new BroadcastStage(neighborRouter); this.validationStage = new ValidationStage(txValidator, recentlySeenBytesCache); - this.receivedStage = new ReceivedStage(tangle, txValidator, snapshotProvider, transactionRequester); + this.receivedStage = new ReceivedStage(tangle, txSolidifier, snapshotProvider, transactionRequester, + milestoneService, config.getCoordinator()); this.batchedHasher = BatchedHasherFactory.create(BatchedHasherFactory.Type.BCTCURL81, 20); this.hashingStage = new HashingStage(batchedHasher); + this.solidifyStage = new SolidifyStage(txSolidifier, tipsViewModel, tangle); + this.txSolidifier = txSolidifier; + this.milestoneSolidifier = milestoneSolidifier; + this.milestoneStage = new MilestoneStage(tangle, milestoneSolidifier, snapshotProvider, + milestoneService, txSolidifier); } @Override @@ -111,6 +130,8 @@ public void start() { addStage("reply", replyStageQueue, replyStage); addStage("received", receivedStageQueue, receivedStage); addStage("broadcast", broadcastStageQueue, broadcastStage); + addStage("solidify", solidifyStageQueue, solidifyStage); + addStage("milestone", milestoneStageQueue, milestoneStage); } /** @@ -126,6 +147,7 @@ private void addStage(String name, BlockingQueue queue, try { while (!Thread.currentThread().isInterrupted()) { ProcessingContext ctx = stage.process(queue.take()); + switch (ctx.getNextStage()) { case REPLY: replyStageQueue.put(ctx); @@ -144,6 +166,12 @@ private void addStage(String name, BlockingQueue queue, case BROADCAST: broadcastStageQueue.put(ctx); break; + case MILESTONE: + milestoneStageQueue.put(ctx); + break; + case SOLIDIFY: + solidifyStageQueue.put(ctx); + break; case ABORT: break; case FINISH: @@ -180,10 +208,16 @@ public BlockingQueue getValidationStageQueue() { return validationStageQueue; } + @Override + public BlockingQueue getMilestoneStageQueue() { + return milestoneStageQueue; + } + @Override public void process(Neighbor neighbor, ByteBuffer data) { try { preProcessStageQueue.put(new ProcessingContext(new PreProcessPayload(neighbor, data))); + refillBroadcastQueue(); } catch (InterruptedException e) { e.printStackTrace(); } @@ -198,6 +232,26 @@ public void process(byte[] txTrits) { hashAndValidate(new ProcessingContext(payload)); } + /** + * Fetches a set of transactions from the {@link TransactionSolidifier} and submits + * the object into the {@link BroadcastStage} queue. + */ + private void refillBroadcastQueue(){ + try{ + Iterator hashIterator = txSolidifier.getBroadcastQueue().iterator(); + Set toRemove = new LinkedHashSet<>(); + while(!Thread.currentThread().isInterrupted() && hashIterator.hasNext()){ + TransactionViewModel tx = hashIterator.next(); + broadcastStageQueue.put(new ProcessingContext(new BroadcastPayload(null, tx))); + toRemove.add(tx); + hashIterator.remove(); + } + txSolidifier.clearFromBroadcastQueue(toRemove); + } catch(InterruptedException e){ + log.info(e.getMessage()); + } + } + /** * Sets up the given hashing stage {@link ProcessingContext} so that up on success, it will submit further to the * validation stage. @@ -255,4 +309,14 @@ public void setBroadcastStage(BroadcastStage broadcastStage) { public void setHashingStage(HashingStage hashingStage) { this.hashingStage = hashingStage; } + + @Override + public void setSolidifyStage(SolidifyStage solidifyStage){ + this.solidifyStage = solidifyStage; + } + + @Override + public void setMilestoneStage(MilestoneStage milestoneStage){ + this.milestoneStage = milestoneStage; + } } diff --git a/src/main/java/com/iota/iri/network/pipeline/ValidationStage.java b/src/main/java/com/iota/iri/network/pipeline/ValidationStage.java index a139210eb4..47c41fa3a9 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ValidationStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ValidationStage.java @@ -1,6 +1,6 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index e3828e9d41..2e96dd9932 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -6,7 +6,10 @@ import com.iota.iri.BundleValidator; import com.iota.iri.IRI; import com.iota.iri.IXI; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; +import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.conf.APIConfig; import com.iota.iri.conf.IotaConfig; import com.iota.iri.controllers.*; @@ -21,7 +24,6 @@ import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.dto.*; import com.iota.iri.service.ledger.LedgerService; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.restserver.RestConnector; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; @@ -108,8 +110,9 @@ public class API { private final TipSelector tipsSelector; private final TipsViewModel tipsViewModel; private final TransactionValidator transactionValidator; - private final LatestMilestoneTracker latestMilestoneTracker; - + private final MilestoneSolidifier milestoneSolidifier; + private final TransactionSolidifier transactionSolidifier; + private final int maxFindTxs; private final int maxRequestList; private final int maxGetTrytes; @@ -148,13 +151,15 @@ public class API { * @param tipsSelector Handles logic for selecting tips based on other transactions * @param tipsViewModel Contains the current tips of this node * @param transactionValidator Validates transactions - * @param latestMilestoneTracker Service that tracks the latest milestone + * @param milestoneSolidifier Service that tracks the latest milestone + * @param transactionSolidifier Holds transaction pipeline, including broadcast transactions */ public API(IotaConfig configuration, IXI ixi, TransactionRequester transactionRequester, - SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, - SnapshotProvider snapshotProvider, LedgerService ledgerService, NeighborRouter neighborRouter, TipSelector tipsSelector, - TipsViewModel tipsViewModel, TransactionValidator transactionValidator, - LatestMilestoneTracker latestMilestoneTracker, TransactionProcessingPipeline txPipeline) { + SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, + SnapshotProvider snapshotProvider, LedgerService ledgerService, NeighborRouter neighborRouter, TipSelector tipsSelector, + TipsViewModel tipsViewModel, TransactionValidator transactionValidator, + MilestoneSolidifier milestoneSolidifier, TransactionProcessingPipeline txPipeline, + TransactionSolidifier transactionSolidifier) { this.configuration = configuration; this.ixi = ixi; @@ -169,8 +174,9 @@ public API(IotaConfig configuration, IXI ixi, TransactionRequester transactionRe this.tipsSelector = tipsSelector; this.tipsViewModel = tipsViewModel; this.transactionValidator = transactionValidator; - this.latestMilestoneTracker = latestMilestoneTracker; - + this.milestoneSolidifier = milestoneSolidifier; + this.transactionSolidifier = transactionSolidifier; + maxFindTxs = configuration.getMaxFindTransactions(); maxRequestList = configuration.getMaxRequestsList(); maxGetTrytes = configuration.getMaxGetTrytes(); @@ -425,7 +431,7 @@ private AbstractResponse checkConsistencyStatement(List transactionsList */ private boolean isNodeSynchronized() { return (snapshotProvider.getLatestSnapshot().getIndex() != snapshotProvider.getInitialSnapshot().getIndex()) && - snapshotProvider.getLatestSnapshot().getIndex() >= latestMilestoneTracker.getLatestMilestoneIndex() -1; + snapshotProvider.getLatestSnapshot().getIndex() >= milestoneSolidifier.getLatestMilestoneIndex() -1; } /** @@ -683,7 +689,7 @@ public AbstractResponse storeTransactionsStatement(List trytes) throws E //store transactions if(transactionViewModel.store(tangle, snapshotProvider.getInitialSnapshot())) { transactionViewModel.setArrivalTime(System.currentTimeMillis()); - transactionValidator.updateStatus(transactionViewModel); + transactionSolidifier.updateStatus(transactionViewModel); transactionViewModel.updateSender("local"); transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "sender"); } @@ -722,8 +728,8 @@ private AbstractResponse getNodeInfoStatement() throws Exception{ Runtime.getRuntime().maxMemory(), Runtime.getRuntime().totalMemory(), - latestMilestoneTracker.getLatestMilestoneHash(), - latestMilestoneTracker.getLatestMilestoneIndex(), + milestoneSolidifier.getLatestMilestoneHash(), + milestoneSolidifier.getLatestMilestoneIndex(), snapshotProvider.getLatestSnapshot().getHash(), snapshotProvider.getLatestSnapshot().getIndex(), diff --git a/src/main/java/com/iota/iri/service/milestone/LatestMilestoneTracker.java b/src/main/java/com/iota/iri/service/milestone/LatestMilestoneTracker.java deleted file mode 100644 index 21cc4043bc..0000000000 --- a/src/main/java/com/iota/iri/service/milestone/LatestMilestoneTracker.java +++ /dev/null @@ -1,106 +0,0 @@ -package com.iota.iri.service.milestone; - -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.Hash; - -/** - *

- * The manager that keeps track of the latest milestone by incorporating a background worker that periodically checks if - * new milestones have arrived. - *

- *

- * Knowing about the latest milestone and being able to compare it to the latest solid milestone allows us to determine - * if our node is "in sync". - *

- */ -public interface LatestMilestoneTracker { - /** - *

- * Returns the index of the latest milestone that was seen by this tracker. - *

- *

- * It returns the internal property that is used to store the latest milestone index. - *

- * - * @return the index of the latest milestone that was seen by this tracker - */ - int getLatestMilestoneIndex(); - - /** - *

- * Returns the transaction hash of the latest milestone that was seen by this tracker. - *

- *

- * It returns the internal property that is used to store the latest milestone index. - *

- * @return the transaction hash of the latest milestone that was seen by this tracker - */ - Hash getLatestMilestoneHash(); - - /** - * Sets the latest milestone. - *

- * It stores the passed in values in their corresponding internal properties and can therefore be used to - * inform the {@link LatestSolidMilestoneTracker} about a new milestone. It is internally used to set the new - * milestone but can also be used by tests to mock a certain behaviour or in case we detect a new milestone in other - * parts of the code. - *

- * - * @param latestMilestoneHash the transaction hash of the milestone - * @param latestMilestoneIndex the milestone index of the milestone - */ - void setLatestMilestone(Hash latestMilestoneHash, int latestMilestoneIndex); - - /** - * Analyzes the given transaction to determine if it is a valid milestone. - *

- * If the transaction that was analyzed represents a milestone, we check if it is younger than the current latest - * milestone and update the internal properties accordingly. - *

- * - * @param transaction the transaction that shall be examined - * @return {@code true} if the milestone could be processed and {@code false} if the bundle is not complete, yet - * @throws MilestoneException if anything unexpected happens while trying to analyze the milestone candidate - */ - boolean processMilestoneCandidate(TransactionViewModel transaction) throws MilestoneException; - - /** - * Does the same as {@link #processMilestoneCandidate(TransactionViewModel)} but automatically retrieves the - * transaction belonging to the passed in hash. - * - * @param transactionHash the hash of the transaction that shall be examined - * @return {@code true} if the milestone could be processed and {@code false} if the bundle is not complete, yet - * @throws MilestoneException if anything unexpected happens while trying to analyze the milestone candidate - */ - boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneException; - - /** - *

- * Since the {@link LatestMilestoneTracker} scans all milestone candidates whenever IRI restarts, this flag gives us - * the ability to determine if this initialization process has finished. - *

- *

- * The values returned by {@link #getLatestMilestoneHash()} and {@link #getLatestMilestoneIndex()} will potentially - * return wrong values until the scan has completed. - *

- * - * @return {@code true} if the initial scan of milestones has finished and {@code false} otherwise - */ - boolean isInitialScanComplete(); - - /** - * This method starts the background worker that automatically calls {@link #processMilestoneCandidate(Hash)} on all - * newly found milestone candidates to update the latest milestone. - */ - void start(); - - /** - * This method stops the background worker that updates the latest milestones. - */ - void shutdown(); - - /** - * Initializes the latest milestone tracker by getting the latest milestone. - */ - void init(); -} diff --git a/src/main/java/com/iota/iri/service/milestone/LatestSolidMilestoneTracker.java b/src/main/java/com/iota/iri/service/milestone/LatestSolidMilestoneTracker.java deleted file mode 100644 index ca99037665..0000000000 --- a/src/main/java/com/iota/iri/service/milestone/LatestSolidMilestoneTracker.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.iota.iri.service.milestone; - -import com.iota.iri.service.snapshot.SnapshotProvider; - -/** - *

- * This interface defines the contract for the manager that keeps track of the latest solid milestone by incorporating a - * background worker that periodically checks for new solid milestones. - *

- *

- * Whenever it finds a new solid milestone that hasn't been applied to the ledger state, yet it triggers the application - * logic which in return updates the {@link SnapshotProvider#getLatestSnapshot()}. Since the latest solid milestone is - * encoded in this latest snapshot of the node, this tracker does not introduce separate getters for the latest solid - * milestone. - *

- */ -public interface LatestSolidMilestoneTracker { - /** - *

- * This method searches for new solid milestones that follow the current latest solid milestone and that have not - * been applied to the ledger state yet and applies them. - *

- *

- * It takes care of applying the solid milestones in the correct order by only allowing solid milestones to be - * applied that are directly following our current latest solid milestone. - *

- * - * @throws MilestoneException if anything unexpected happens while updating the latest solid milestone - */ - void trackLatestSolidMilestone() throws MilestoneException; - - /** - * This method starts the background worker that automatically calls {@link #trackLatestSolidMilestone()} - * periodically to keep the latest solid milestone up to date. - */ - void start(); - - /** - * This method stops the background worker that automatically updates the latest solid milestone. - */ - void shutdown(); -} diff --git a/src/main/java/com/iota/iri/service/milestone/MilestoneService.java b/src/main/java/com/iota/iri/service/milestone/MilestoneService.java index 91559d38c8..dc4f495a42 100644 --- a/src/main/java/com/iota/iri/service/milestone/MilestoneService.java +++ b/src/main/java/com/iota/iri/service/milestone/MilestoneService.java @@ -83,7 +83,7 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i *

*

* This allows us to reprocess the milestone in case of errors where the given milestone could not be applied to the - * ledger state. It is for example used by the automatic repair routine of the {@link LatestSolidMilestoneTracker} + * ledger state. It is for example used by the automatic repair routine of the {@link MilestoneSolidifier} * (to recover from inconsistencies due to crashes of IRI). *

*

@@ -104,7 +104,7 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i *

* We determine if the transaction was confirmed by examining its {@code snapshotIndex} value. For this method to * work we require that the previous milestones have been processed already (which is enforced by the {@link - * com.iota.iri.service.milestone.LatestSolidMilestoneTracker} which applies the milestones in the order that they + * com.iota.iri.service.milestone.MilestoneSolidifier} which applies the milestones in the order that they * are issued by the coordinator). *

* diff --git a/src/main/java/com/iota/iri/service/milestone/MilestoneSolidifier.java b/src/main/java/com/iota/iri/service/milestone/MilestoneSolidifier.java index 453c9bee3d..584ce9922a 100644 --- a/src/main/java/com/iota/iri/service/milestone/MilestoneSolidifier.java +++ b/src/main/java/com/iota/iri/service/milestone/MilestoneSolidifier.java @@ -1,7 +1,11 @@ package com.iota.iri.service.milestone; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +import java.util.List; +import java.util.Map; + /** * This interface defines the contract for a manager that tries to solidify unsolid milestones by incorporating a * background worker that periodically checks the solidity of the milestones and issues transaction requests for the @@ -25,4 +29,64 @@ public interface MilestoneSolidifier { * This method shuts down the background worker that asynchronously solidifies the milestones. */ void shutdown(); + + /** + * This method returns the latest milestone index. + * @return Latest Milestone Index + */ + int getLatestMilestoneIndex(); + + /** + * This method returns the latest milestone hash. + * @return Latest Milestone Hash + */ + Hash getLatestMilestoneHash(); + + /** + * Checks if the MilestoneSolidifier has been initialised fully. This registers true if the solidifier has performed + * an initial solidification of present elements in the db. + * + * @return Initialised state. + */ + boolean isInitialScanComplete(); + + /** + * Add to the seen milestones queue to be held for processing solid milestones. In order to be added to the queue + * the transaction must pass {@link MilestoneService#validateMilestone(TransactionViewModel, int)}. + * + * @param milestoneHash The {@link Hash} of the seen milestone + * @param milestoneIndex The index of the seen milestone + */ + void addSeenMilestone(Hash milestoneHash, int milestoneIndex); + + /** + * Removes the specified element from the unsolid milestones queue. In the event that a milestone manages to get + * into the queue, but is later determined to be {@link MilestoneValidity#INVALID}, this method allows it to be + * removed. + * + * @param milestoneHash The {@link Hash} of the milestone to be removed + */ + void removeFromQueue(Hash milestoneHash); + + /** + * Set the latest milestone seen by the node. + * + * @param milestoneHash The latest milestone hash + * @param milestoneIndex The latest milestone index + */ + void setLatestMilestone(Hash milestoneHash, int milestoneIndex); + + /** + * Returns the oldest current milestone object in the unsolid milestone queue to be processed for solidification. + * @return The oldest milestone hash to index mapping + */ + List> getOldestMilestonesInQueue(); + + /** + * Set the latest milestone hash and index, publish and log the update. + * @param oldMilestoneIndex Previous milestone index + * @param newMilestoneIndex New milestone index + * @param newMilestoneHash New milestone hash + */ + void logNewMilestone(int oldMilestoneIndex, int newMilestoneIndex, Hash newMilestoneHash); } diff --git a/src/main/java/com/iota/iri/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/com/iota/iri/service/milestone/impl/LatestMilestoneTrackerImpl.java deleted file mode 100644 index f25475637a..0000000000 --- a/src/main/java/com/iota/iri/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ /dev/null @@ -1,392 +0,0 @@ -package com.iota.iri.service.milestone.impl; - -import com.iota.iri.conf.IotaConfig; -import com.iota.iri.controllers.AddressViewModel; -import com.iota.iri.controllers.MilestoneViewModel; -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.Hash; -import com.iota.iri.service.milestone.LatestMilestoneTracker; -import com.iota.iri.service.milestone.MilestoneException; -import com.iota.iri.service.milestone.MilestoneService; -import com.iota.iri.service.milestone.MilestoneSolidifier; -import com.iota.iri.service.snapshot.Snapshot; -import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.log.interval.IntervalLogger; -import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; -import com.iota.iri.utils.thread.SilentScheduledExecutorService; - -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.HashSet; -import java.util.Set; -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - *

- * Creates a tracker that automatically detects new milestones by incorporating a background worker that periodically - * checks all transactions that are originating from the coordinator address and that exposes the found latest milestone - * via getters. - *

- *

- * It can be used to determine the sync-status of the node by comparing these values against the latest solid - * milestone. - *

- */ -public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { - /** - * Holds the amount of milestone candidates that will be analyzed per iteration of the background worker. - */ - private static final int MAX_CANDIDATES_TO_ANALYZE = 5000; - - /** - * Holds the time (in milliseconds) between iterations of the background worker. - */ - private static final int RESCAN_INTERVAL = 1000; - - /** - * Holds the logger of this class (a rate limited logger than doesn't spam the CLI output). - */ - private static final IntervalLogger log = new IntervalLogger(LatestMilestoneTrackerImpl.class); - - /** - * Holds the Tangle object which acts as a database interface. - */ - private final Tangle tangle; - - /** - * The snapshot provider which gives us access to the relevant snapshots that the node uses (for faster - * bootstrapping). - */ - private final SnapshotProvider snapshotProvider; - - /** - * Service class containing the business logic of the milestone package. - */ - private final MilestoneService milestoneService; - - /** - * Holds a reference to the manager that takes care of solidifying milestones. - */ - private final MilestoneSolidifier milestoneSolidifier; - - /** - * Holds the coordinator address which is used to filter possible milestone candidates. - */ - private final Hash coordinatorAddress; - - /** - * Holds a reference to the manager of the background worker. - */ - private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( - "Latest Milestone Tracker", log.delegate()); - - /** - * Holds the milestone index of the latest milestone that we have seen / processed. - */ - private int latestMilestoneIndex; - - /** - * Holds the transaction hash of the latest milestone that we have seen / processed. - */ - private Hash latestMilestoneHash; - - /** - * A set that allows us to keep track of the candidates that have been seen and added to the {@link - * #milestoneCandidatesToAnalyze} already. - */ - private final Set seenMilestoneCandidates = new HashSet<>(); - - /** - * A list of milestones that still have to be analyzed. - */ - private final Deque milestoneCandidatesToAnalyze = new ArrayDeque<>(); - - /** - * A flag that allows us to detect if the background worker is in its first iteration (for different log - * handling). - */ - private boolean firstRun = true; - - /** - * Flag which indicates if this tracker has finished its initial scan of all old milestone candidates. - */ - private boolean initialized = false; - - /** - * @param tangle Tangle object which acts as a database interface - * @param snapshotProvider manager for the snapshots that allows us to retrieve the relevant snapshots of this node - * @param milestoneService contains the important business logic when dealing with milestones - * @param milestoneSolidifier manager that takes care of solidifying milestones - * @param config configuration object which allows us to determine the important config parameters of the node - */ - public LatestMilestoneTrackerImpl(Tangle tangle, SnapshotProvider snapshotProvider, MilestoneService milestoneService, - MilestoneSolidifier milestoneSolidifier, IotaConfig config) { - this.tangle = tangle; - this.snapshotProvider = snapshotProvider; - this.milestoneService = milestoneService; - this.milestoneSolidifier = milestoneSolidifier; - this.coordinatorAddress = config.getCoordinator(); - } - - @Override - public void init() { - bootstrapLatestMilestoneValue(); - } - - /** - * {@inheritDoc} - * - *

- * In addition to setting the internal properties, we also issue a log message and publish the change to the ZeroMQ - * message processor so external receivers get informed about this change. - *

- */ - @Override - public void setLatestMilestone(Hash latestMilestoneHash, int latestMilestoneIndex) { - tangle.publish("lmi %d %d", this.latestMilestoneIndex, latestMilestoneIndex); - log.delegate().info("Latest milestone has changed from #" + this.latestMilestoneIndex + " to #" + - latestMilestoneIndex); - - this.latestMilestoneHash = latestMilestoneHash; - this.latestMilestoneIndex = latestMilestoneIndex; - } - - @Override - public int getLatestMilestoneIndex() { - return latestMilestoneIndex; - } - - @Override - public Hash getLatestMilestoneHash() { - return latestMilestoneHash; - } - - @Override - public boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneException { - try { - return processMilestoneCandidate(TransactionViewModel.fromHash(tangle, transactionHash)); - } catch (Exception e) { - throw new MilestoneException("unexpected error while analyzing the transaction " + transactionHash, e); - } - } - - /** - * {@inheritDoc} - */ - @Override - public boolean processMilestoneCandidate(TransactionViewModel transaction) throws MilestoneException { - try { - if (coordinatorAddress.equals(transaction.getAddressHash()) && - transaction.getCurrentIndex() == 0) { - - int milestoneIndex = milestoneService.getMilestoneIndex(transaction); - - // if the milestone is older than our ledger start point: we already processed it in the past - if (milestoneIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { - return true; - } - - switch (milestoneService.validateMilestone(transaction, milestoneIndex)) { - case VALID: - if (milestoneIndex > latestMilestoneIndex) { - setLatestMilestone(transaction.getHash(), milestoneIndex); - } - - if (!transaction.isSolid()) { - milestoneSolidifier.add(transaction.getHash(), milestoneIndex); - } - - transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); - break; - - case INCOMPLETE: - return false; - - case INVALID: - // do not re-analyze anymore - return true; - - default: - // we can consider the milestone candidate processed and move on w/o farther action - } - } - - return true; - } catch (Exception e) { - throw new MilestoneException("unexpected error while analyzing the " + transaction, e); - } - } - - @Override - public boolean isInitialScanComplete() { - return initialized; - } - - /** - * {@inheritDoc} - *

- * We repeatedly call {@link #latestMilestoneTrackerThread()} to actively look for new milestones in our database. - * This is a bit inefficient and should at some point maybe be replaced with a check on transaction arrival, but - * this would required adjustments in the whole way IRI handles transactions and is therefore postponed for - * now. - *

- */ - @Override - public void start() { - executorService.silentScheduleWithFixedDelay(this::latestMilestoneTrackerThread, 0, RESCAN_INTERVAL, - TimeUnit.MILLISECONDS); - } - - @Override - public void shutdown() { - executorService.shutdownNow(); - } - - /** - *

- * This method contains the logic for scanning for new latest milestones that gets executed in a background - * worker. - *

- *

- * It first collects all new milestone candidates that need to be analyzed, then analyzes them and finally checks if - * the initialization is complete. In addition to this scanning logic it also issues regular log messages about the - * progress of the scanning. - *

- */ - private void latestMilestoneTrackerThread() { - try { - logProgress(); - collectNewMilestoneCandidates(); - - // additional log message on the first run to indicate how many milestone candidates we have in total - if (firstRun) { - firstRun = false; - - logProgress(); - } - - analyzeMilestoneCandidates(); - checkIfInitializationComplete(); - } catch (MilestoneException e) { - log.error("error while analyzing the milestone candidates", e); - } - } - - /** - *

- * This method emits a log message about the scanning progress. - *

- *

- * It only emits a log message if we have more than one {@link #milestoneCandidatesToAnalyze}, which means that the - * very first call to this method in the "first run" on {@link #latestMilestoneTrackerThread()} will not produce any - * output (which is the reason why we call this method a second time after we have collected all the - * candidates in the "first run"). - *

- */ - private void logProgress() { - if (milestoneCandidatesToAnalyze.size() > 1) { - log.info("Processing milestone candidates (" + milestoneCandidatesToAnalyze.size() + " remaining) ..."); - } - } - - /** - *

- * This method collects the new milestones that have not been "seen" before, by collecting them in the {@link - * #milestoneCandidatesToAnalyze} queue. - *

- *

- * We simply request all transaction that are originating from the coordinator address and treat them as potential - * milestone candidates. - *

- * - * @throws MilestoneException if anything unexpected happens while collecting the new milestone candidates - */ - private void collectNewMilestoneCandidates() throws MilestoneException { - try { - List transactions = AddressViewModel.loadAsSortedList(tangle, coordinatorAddress); - for (TransactionViewModel tvm : transactions) { - if (Thread.currentThread().isInterrupted()) { - return; - } - - if (tvm != null && seenMilestoneCandidates.add(tvm)) { - milestoneCandidatesToAnalyze.addFirst(tvm); - } - } - } catch (Exception e) { - throw new MilestoneException("failed to collect the new milestone candidates", e); - } - } - - /** - *

- * This method analyzes the milestone candidates by working through the {@link #milestoneCandidatesToAnalyze} - * queue. - *

- *

- * We only process {@link #MAX_CANDIDATES_TO_ANALYZE} at a time, to give the caller the option to terminate early - * and pick up new milestones as fast as possible without being stuck with analyzing the old ones for too - * long. - *

- * - * @throws MilestoneException if anything unexpected happens while analyzing the milestone candidates - */ - private void analyzeMilestoneCandidates() throws MilestoneException { - int candidatesToAnalyze = Math.min(milestoneCandidatesToAnalyze.size(), MAX_CANDIDATES_TO_ANALYZE); - for (int i = 0; i < candidatesToAnalyze; i++) { - if (Thread.currentThread().isInterrupted()) { - return; - } - - TransactionViewModel candidateTransactionViewModel = milestoneCandidatesToAnalyze.pollFirst(); - if(!processMilestoneCandidate(candidateTransactionViewModel)) { - seenMilestoneCandidates.remove(candidateTransactionViewModel); - } - } - } - - /** - *

- * This method checks if the initialization is complete. - *

- *

- * It simply checks if the {@link #initialized} flag is not set yet and there are no more {@link - * #milestoneCandidatesToAnalyze}. If the initialization was complete, we issue a log message and set the - * corresponding flag to {@code true}. - *

- */ - private void checkIfInitializationComplete() { - if (!initialized && milestoneCandidatesToAnalyze.size() == 0) { - initialized = true; - - log.info("Processing milestone candidates ... [DONE]").triggerOutput(true); - } - } - - /** - *

- * This method bootstraps this tracker with the latest milestone values that can easily be retrieved without - * analyzing any transactions (for faster startup). - *

- *

- * It first sets the latest milestone to the values found in the latest snapshot and then check if there is a younger - * milestone at the end of our database. While this last entry in the database doesn't necessarily have to be the - * latest one we know it at least gives a reasonable value most of the times. - *

- */ - private void bootstrapLatestMilestoneValue() { - Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); - setLatestMilestone(latestSnapshot.getHash(), latestSnapshot.getIndex()); - - try { - MilestoneViewModel lastMilestoneInDatabase = MilestoneViewModel.latest(tangle); - if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getLatestMilestoneIndex()) { - setLatestMilestone(lastMilestoneInDatabase.getHash(), lastMilestoneInDatabase.index()); - } - } catch (Exception e) { - log.error("unexpectedly failed to retrieve the latest milestone from the database", e); - } - } -} diff --git a/src/main/java/com/iota/iri/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/com/iota/iri/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java deleted file mode 100644 index 9ae95775a3..0000000000 --- a/src/main/java/com/iota/iri/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ /dev/null @@ -1,471 +0,0 @@ -package com.iota.iri.service.milestone.impl; - -import com.iota.iri.conf.SolidificationConfig; -import com.iota.iri.controllers.MilestoneViewModel; -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.Hash; -import com.iota.iri.network.TransactionRequester; -import com.iota.iri.service.ledger.LedgerService; -import com.iota.iri.service.milestone.LatestMilestoneTracker; -import com.iota.iri.service.milestone.MilestoneException; -import com.iota.iri.service.milestone.MilestoneService; -import com.iota.iri.service.milestone.LatestSolidMilestoneTracker; -import com.iota.iri.service.snapshot.Snapshot; -import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.ASCIIProgressBar; -import com.iota.iri.utils.log.interval.IntervalLogger; -import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; -import com.iota.iri.utils.thread.SilentScheduledExecutorService; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; - -/** - *

- * Creates a manager that keeps track of the latest solid milestones and that triggers the application of these - * milestones and their corresponding balance changes to the latest {@link Snapshot} by incorporating a background - * worker that periodically checks for new solid milestones. - *

- *

- * It extends this with a mechanisms to recover from database corruptions by using a backoff strategy that reverts the - * changes introduced by previous milestones whenever an error is detected until the problem causing milestone was - * found. - *

- */ -public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTracker { - /** - * Holds the interval (in milliseconds) in which the {@link #trackLatestSolidMilestone()} method gets - * called by the background worker. - */ - private static final int RESCAN_INTERVAL = 5000; - - /** - * Holds the logger of this class (a rate limited logger than doesn't spam the CLI output). - */ - private static final IntervalLogger log = new IntervalLogger(LatestSolidMilestoneTrackerImpl.class); - - /** - * Holds the Tangle object which acts as a database interface. - */ - private final Tangle tangle; - - /** - * The snapshot provider which gives us access to the relevant snapshots that the node uses (for the ledger - * state). - */ - private final SnapshotProvider snapshotProvider; - - /** - * Holds a reference to the service instance containing the business logic of the milestone package. - */ - private final MilestoneService milestoneService; - - /** - * Holds a reference to the manager that keeps track of the latest milestone. - */ - private final LatestMilestoneTracker latestMilestoneTracker; - - /** - * Holds a reference to the service that contains the logic for applying milestones to the ledger state. - */ - private final LedgerService ledgerService; - - /** - * Holds a reference to the manager of the background worker. - */ - private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( - "Latest Solid Milestone Tracker", log.delegate()); - - /** - * Boolean flag that is used to identify the first iteration of the background worker. - */ - private boolean firstRun = true; - - /** - * Holds the milestone index of the milestone that caused the repair logic to get started. - */ - private int errorCausingMilestoneIndex = Integer.MAX_VALUE; - - /** - * Counter for the backoff repair strategy (see {@link #repairCorruptedMilestone(MilestoneViewModel)}. - */ - private int repairBackoffCounter = 0; - - /** - * Uses to clear the request queue once we are fully synced. - */ - private TransactionRequester transactionRequester; - - /** - * Holds information about the current synchronisation progress. - */ - private SyncProgressInfo syncProgressInfo = new SyncProgressInfo(); - - private SolidificationConfig solidificationConfig; - - /** - *

- * This method initializes the instance and registers its dependencies. - *

- *

- * It stores the passed in values in their corresponding private properties. - *

- *

- * Note: Instead of handing over the dependencies in the constructor, we register them lazy. This allows us to have - * circular dependencies because the instantiation is separated from the dependency injection. To reduce the - * amount of code that is necessary to correctly instantiate this class, we return the instance itself which - * allows us to still instantiate, initialize and assign in one line - see Example: - *

- * {@code latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl().init(...);} - * - * @param tangle Tangle object which acts as a database interface - * @param snapshotProvider manager for the snapshots that allows us to retrieve the relevant snapshots of this node - * @param milestoneService contains the important business logic when dealing with milestones - * @param ledgerService the manager for - * @param latestMilestoneTracker the manager that keeps track of the latest milestone - * @param transactionRequester the manager which keeps and tracks transactions which are requested - * @return the initialized instance itself to allow chaining - */ - public LatestSolidMilestoneTrackerImpl(Tangle tangle, SnapshotProvider snapshotProvider, - MilestoneService milestoneService, LedgerService ledgerService, - LatestMilestoneTracker latestMilestoneTracker, TransactionRequester transactionRequester, - SolidificationConfig solidificationConfig) { - - this.tangle = tangle; - this.snapshotProvider = snapshotProvider; - this.milestoneService = milestoneService; - this.ledgerService = ledgerService; - this.latestMilestoneTracker = latestMilestoneTracker; - this.transactionRequester = transactionRequester; - this.solidificationConfig = solidificationConfig; - } - - @Override - public void start() { - executorService.silentScheduleWithFixedDelay(this::latestSolidMilestoneTrackerThread, 0, RESCAN_INTERVAL, - TimeUnit.MILLISECONDS); - } - - @Override - public void shutdown() { - executorService.shutdownNow(); - } - - /** - * {@inheritDoc} - * - *

- * In addition to applying the found milestones to the ledger state it also issues log messages and keeps the - * {@link LatestMilestoneTracker} in sync (if we happen to process a new latest milestone faster). - *

- */ - @Override - public void trackLatestSolidMilestone() throws MilestoneException { - try { - int currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); - if (currentSolidMilestoneIndex < latestMilestoneTracker.getLatestMilestoneIndex()) { - MilestoneViewModel nextMilestone; - while (!Thread.currentThread().isInterrupted() && - (nextMilestone = MilestoneViewModel.get(tangle, currentSolidMilestoneIndex + 1)) != null && - TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) { - - syncLatestMilestoneTracker(nextMilestone.getHash(), nextMilestone.index()); - applySolidMilestoneToLedger(nextMilestone); - logChange(currentSolidMilestoneIndex); - - currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); - if (currentSolidMilestoneIndex == latestMilestoneTracker.getLatestMilestoneIndex()) { - transactionRequester.clearRecentlyRequestedTransactions(); - } - } - } else { - syncLatestMilestoneTracker(snapshotProvider.getLatestSnapshot().getHash(), - currentSolidMilestoneIndex); - } - } catch (Exception e) { - throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); - } - } - - /** - *

- * Contains the logic for the background worker. - *

- *

- * It simply calls {@link #trackLatestSolidMilestone()} and wraps with a log handler that prevents the {@link - * MilestoneException} to crash the worker. - *

- */ - private void latestSolidMilestoneTrackerThread() { - try { - if (firstRun) { - firstRun = false; - - ledgerService.restoreLedgerState(); - int milestoneStartIndex = snapshotProvider.getInitialSnapshot().getIndex(); - syncProgressInfo.setSyncMilestoneStartIndex(milestoneStartIndex); - logChange(milestoneStartIndex); - } - - trackLatestSolidMilestone(); - } catch (Exception e) { - log.error("error while updating the solid milestone", e); - } - } - - /** - *

- * Applies the given milestone to the ledger. - *

- *

- * If the application of the milestone fails, we start a repair routine which will revert the milestones preceding - * our current milestone and consequently try to reapply them in the next iteration of the {@link - * #trackLatestSolidMilestone()} method (until the problem is solved). - *

- * - * @param milestone the milestone that shall be applied to the ledger state - * @throws Exception if anything unexpected goes wrong while applying the milestone to the ledger - */ - private void applySolidMilestoneToLedger(MilestoneViewModel milestone) throws Exception { - if (ledgerService.applyMilestoneToLedger(milestone)) { - if (isRepairRunning() && isRepairSuccessful(milestone)) { - stopRepair(); - } - syncProgressInfo.addMilestoneApplicationTime(); - } else { - repairCorruptedMilestone(milestone); - } - } - - /** - *

- * Checks if we are currently trying to repair a milestone. - *

- *

- * We simply use the {@link #repairBackoffCounter} as an indicator if a repair routine is running. - *

- * - * @return {@code true} if we are trying to repair a milestone and {@code false} otherwise - */ - private boolean isRepairRunning() { - return repairBackoffCounter != 0; - } - - /** - *

- * Checks if we successfully repaired the corrupted milestone. - *

- *

- * To determine if the repair routine was successful we check if the processed milestone has a higher index than the - * one that initially could not get applied to the ledger. - *

- * - * @param processedMilestone the currently processed milestone - * @return {@code true} if we advanced to a milestone following the corrupted one and {@code false} otherwise - */ - private boolean isRepairSuccessful(MilestoneViewModel processedMilestone) { - return processedMilestone.index() > errorCausingMilestoneIndex; - } - - /** - *

- * Resets the internal variables that are used to keep track of the repair process. - *

- *

- * It gets called whenever we advance to a milestone that has a higher milestone index than the milestone that - * initially caused the repair routine to kick in (see {@link #repairCorruptedMilestone(MilestoneViewModel)}. - *

- */ - private void stopRepair() { - repairBackoffCounter = 0; - errorCausingMilestoneIndex = Integer.MAX_VALUE; - } - - /** - *

- * Keeps the {@link LatestMilestoneTracker} in sync with our current progress. - *

- *

- * Since the {@link LatestMilestoneTracker} scans all old milestones during its startup (which can take a while to - * finish) it can happen that we see a newer latest milestone faster than this manager. - *

- * Note: This method ensures that the latest milestone index is always bigger or equals the latest solid milestone - * index. - * - * @param milestoneHash transaction hash of the milestone - * @param milestoneIndex milestone index - */ - private void syncLatestMilestoneTracker(Hash milestoneHash, int milestoneIndex) { - if (milestoneIndex > latestMilestoneTracker.getLatestMilestoneIndex()) { - latestMilestoneTracker.setLatestMilestone(milestoneHash, milestoneIndex); - } - } - - /** - *

- * Emits a log message whenever the latest solid milestone changes. - *

- *

- * It simply compares the current latest milestone index against the previous milestone index and emits the log - * messages using the {@link #log} and the {@link #messageQ} instances if it differs. - *

- * - * @param prevSolidMilestoneIndex the milestone index before the change - */ - private void logChange(int prevSolidMilestoneIndex) { - Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); - int latestSolidMilestoneIndex = latestSnapshot.getIndex(); - - if (prevSolidMilestoneIndex == latestSolidMilestoneIndex) { - return; - } - - Hash latestMilestoneHash = latestSnapshot.getHash(); - log.info("Latest SOLID milestone index changed from #" + prevSolidMilestoneIndex + " to #" + latestSolidMilestoneIndex); - - tangle.publish("lmsi %d %d", prevSolidMilestoneIndex, latestSolidMilestoneIndex); - tangle.publish("lmhs %s", latestMilestoneHash); - - if (!solidificationConfig.isPrintSyncProgressEnabled()) { - return; - } - - int latestMilestoneIndex = latestMilestoneTracker.getLatestMilestoneIndex(); - - // only print more sophisticated progress if we are coming from a more unsynced state - if(latestMilestoneIndex - latestSolidMilestoneIndex < 1) { - syncProgressInfo.setSyncMilestoneStartIndex(latestSolidMilestoneIndex); - syncProgressInfo.resetMilestoneApplicationTimes(); - return; - } - - int estSecondsToBeSynced = syncProgressInfo.computeEstimatedTimeToSyncUpSeconds(latestMilestoneIndex, - latestSolidMilestoneIndex); - StringBuilder progressSB = new StringBuilder(); - - // add progress bar - progressSB.append(ASCIIProgressBar.getProgressBarString(syncProgressInfo.getSyncMilestoneStartIndex(), - latestMilestoneIndex, latestSolidMilestoneIndex)); - // add lsm to lm - progressSB.append(String.format(" [LSM %d / LM %d - remaining: %d]", latestSolidMilestoneIndex, - latestMilestoneIndex, latestMilestoneIndex - latestSolidMilestoneIndex)); - // add estimated time to get fully synced - if (estSecondsToBeSynced != -1) { - progressSB.append(String.format(" - est. seconds to get synced: %d", estSecondsToBeSynced)); - } - log.info(progressSB.toString()); - } - - - /** - *

- * Tries to actively repair the ledger by reverting the milestones preceding the given milestone. - *

- *

- * It gets called when a milestone could not be applied to the ledger state because of problems like "inconsistent - * balances". While this should theoretically never happen (because milestones are by definition "consistent"), it - * can still happen because IRI crashed or got stopped in the middle of applying a milestone or if a milestone - * was processed in the wrong order. - *

- *

- * Every time we call this method the internal {@link #repairBackoffCounter} is incremented which causes the next - * call of this method to repair an additional milestone. This means that whenever we face an error we first try to - * reset only the last milestone, then the two last milestones, then the three last milestones (and so on ...) until - * the problem was fixed. - *

- *

- * To be able to tell when the problem is fixed and the {@link #repairBackoffCounter} can be reset, we store the - * milestone index that caused the problem the first time we call this method. - *

- * - * @param errorCausingMilestone the milestone that failed to be applied - * @throws MilestoneException if we failed to reset the corrupted milestone - */ - private void repairCorruptedMilestone(MilestoneViewModel errorCausingMilestone) throws MilestoneException { - if (repairBackoffCounter++ == 0) { - errorCausingMilestoneIndex = errorCausingMilestone.index(); - } - for (int i = errorCausingMilestone.index(); i > errorCausingMilestone.index() - repairBackoffCounter; i--) { - milestoneService.resetCorruptedMilestone(i); - } - } - - /** - * Holds variables containing information needed for sync progress calculation. - */ - private static class SyncProgressInfo { - /** - * The actual start milestone index from which the node started from when syncing up. - */ - private int syncMilestoneStartIndex; - - /** - * Used to calculate the average time needed to apply a milestone. - */ - private BlockingQueue lastMilestoneApplyTimes = new ArrayBlockingQueue<>(100); - - /** - * Gets the milestone sync start index. - * - * @return the milestone sync start index - */ - int getSyncMilestoneStartIndex() { - return syncMilestoneStartIndex; - } - - /** - * Sets the milestone sync start index. - * - * @param syncMilestoneStartIndex the value to set - */ - void setSyncMilestoneStartIndex(int syncMilestoneStartIndex) { - this.syncMilestoneStartIndex = syncMilestoneStartIndex; - } - - /** - * Adds the current time as a time where a milestone got applied to the ledger state. - */ - void addMilestoneApplicationTime(){ - if(lastMilestoneApplyTimes.remainingCapacity() == 0){ - lastMilestoneApplyTimes.remove(); - } - lastMilestoneApplyTimes.add(System.currentTimeMillis()); - } - - /** - * Clears the records of times when milestones got applied. - */ - void resetMilestoneApplicationTimes(){ - lastMilestoneApplyTimes.clear(); - } - - /** - * Computes the estimated time (seconds) needed to get synced up by looking at the last N milestone applications - * times and the current solid and last milestone indices. - * - * @param latestMilestoneIndex the current latest known milestone index - * @param latestSolidMilestoneIndex the current to the ledger applied milestone index - * @return the number of seconds needed to get synced up or -1 if not enough data is available - */ - int computeEstimatedTimeToSyncUpSeconds(int latestMilestoneIndex, int latestSolidMilestoneIndex) { - // compute average time needed to apply a milestone - Object[] times = lastMilestoneApplyTimes.toArray(); - long sumDelta = 0; - double avgMilestoneApplyMillisec; - if (times.length > 1) { - // compute delta sum - for (int i = times.length - 1; i > 1; i--) { - sumDelta += ((Long)times[i]) - ((Long)times[i - 1]); - } - avgMilestoneApplyMillisec = (double) sumDelta / (double) (times.length - 1); - } else { - return -1; - } - - return (int) ((avgMilestoneApplyMillisec / 1000) * (latestMilestoneIndex - latestSolidMilestoneIndex)); - } - } - -} diff --git a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java index 9706f1f9df..cd13ee7d67 100644 --- a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java @@ -1,351 +1,667 @@ package com.iota.iri.service.milestone.impl; -import com.iota.iri.TransactionValidator; +import com.iota.iri.conf.SolidificationConfig; +import com.iota.iri.controllers.AddressViewModel; +import com.iota.iri.controllers.MilestoneViewModel; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.ledger.LedgerService; +import com.iota.iri.service.milestone.MilestoneException; +import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.milestone.MilestoneValidity; +import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.storage.Tangle; +import com.iota.iri.utils.ASCIIProgressBar; import com.iota.iri.utils.log.interval.IntervalLogger; import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; import com.iota.iri.utils.thread.SilentScheduledExecutorService; import java.util.*; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -/** - *

- * This class implements the basic contract of the {@link MilestoneSolidifier} interface. - *

- *

- * It manages a map of unsolid milestones to collect all milestones that have to be solidified. It then periodically - * issues checkSolidity calls on the earliest milestones to solidify them. - *

- *

- * To save resources and make the call a little bit more efficient, we cache the earliest milestones in a separate map, - * so the relatively expensive task of having to search for the next earliest milestone in the pool only has to be - * performed after a milestone has become solid or irrelevant for our node. - *

- */ +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ArrayBlockingQueue; + public class MilestoneSolidifierImpl implements MilestoneSolidifier { - /** - * Defines the amount of milestones that we "simultaneously" try to solidify in one pass. - */ - private static final int SOLIDIFICATION_QUEUE_SIZE = 20; + private static final IntervalLogger log = new IntervalLogger(MilestoneSolidifierImpl.class); + // Max size fo the solidification queue + private static final int MAX_SIZE = 10; - /** - * Defines the interval in which solidity checks are issued (in milliseconds). - */ - private static final int SOLIDIFICATION_INTERVAL = 500; + private Map unsolidMilestones = new ConcurrentHashMap<>(); + private Map solidificationQueue = new ConcurrentHashMap<>(); + private Map seenMilestones = new ConcurrentHashMap<>(); + + private Map.Entry oldestMilestoneInQueue = null; + + private int latestMilestoneIndex; + private Hash latestMilestoneHash; + private int latestSolidMilestone; + + private TransactionSolidifier transactionSolidifier; + private LedgerService ledgerService; + private SnapshotProvider snapshotProvider; + private Tangle tangle; + private TransactionRequester transactionRequester; + private MilestoneService milestoneService; + private SolidificationConfig config; - /** - *

- * Defines the maximum amount of transactions that are allowed to get processed while trying to solidify a - * milestone. - *

- *

- * Note: We want to find the next previous milestone and not get stuck somewhere at the end of the tangle with a - * long running {@link TransactionValidator#checkSolidity(Hash)} call. - *

- */ - private static final int SOLIDIFICATION_TRANSACTIONS_LIMIT = 50000; + + private SyncProgressInfo syncProgressInfo = new SyncProgressInfo(); /** - * Logger for this class allowing us to dump debug and status messages. + * Holds the milestone index of the milestone that caused the repair logic to get started. */ - private static final IntervalLogger log = new IntervalLogger(MilestoneSolidifier.class); + private int errorCausingMilestoneIndex = Integer.MAX_VALUE; /** - * Holds the snapshot provider which gives us access to the relevant snapshots. + * Counter for the backoff repair strategy (see {@link #repairCorruptedMilestone(MilestoneViewModel)}. */ - private final SnapshotProvider snapshotProvider; + private int repairBackoffCounter = 0; /** - * Holds a reference to the TransactionValidator which allows us to issue solidity checks. + * An indicator for whether or not the solidifier has processed from the first batch of seen milestones */ - private final TransactionValidator transactionValidator; + private boolean initialized = false; /** - * Holds a reference to the manager of the background worker. + * The execution service for the milestone solidifier thread */ private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( - "Milestone Solidifier", log.delegate()); + "Milestone Solidifier"); + + private boolean firstRun = true; + /** - *

- * Holds the milestones that were newly added, but not examined yet. - *

- *

- * Note: This is used to be able to add milestones to the solidifier without having to synchronize the access to the - * underlying Maps. - *

+ * Constructor for the {@link MilestoneSolidifierImpl}. This class holds milestone objects to be processed for + * solidification. It also tracks the latest solid milestone object. + * + * @param transactionSolidifier Solidification service for transaction objects + * @param tangle DB reference + * @param snapshotProvider SnapshotProvider for fetching local snapshots + * @param ledgerService LedgerService for bootstrapping ledger state + * @param txRequester TransationRequester to submit missing transactions to + * @param milestoneService MilestoneService for resetting corrupted milestones + * @param config Configuration to determine if additional logging will be used */ - private final Map newlyAddedMilestones = new ConcurrentHashMap<>(); + public MilestoneSolidifierImpl(TransactionSolidifier transactionSolidifier, Tangle tangle, + SnapshotProvider snapshotProvider, LedgerService ledgerService, + TransactionRequester txRequester, MilestoneService milestoneService, + SolidificationConfig config){ + this.transactionSolidifier = transactionSolidifier; + this.snapshotProvider = snapshotProvider; + this.ledgerService = ledgerService; + this.tangle = tangle; + this.transactionRequester = txRequester; + this.milestoneService = milestoneService; + this.config = config; + } + + @Override + public void start(){ + executorService.silentExecute(this::milestoneSolidificationThread); + } + + @Override + public void shutdown(){ + executorService.shutdownNow(); + } + + + private void milestoneSolidificationThread() { + while(!Thread.currentThread().isInterrupted()) { + try { + if (firstRun) { + firstRun = false; + latestMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); + latestMilestoneHash = snapshotProvider.getLatestSnapshot().getHash(); + + bootStrapSolidMilestones(); + ledgerService.restoreLedgerState(); + logChange(snapshotProvider.getInitialSnapshot().getIndex()); + + syncProgressInfo.setSyncMilestoneStartIndex(snapshotProvider.getInitialSnapshot().getIndex()); + } + solidifySeenMilestones(); + processSolidifyQueue(); + + checkLatestSolidMilestone(); + solidifyLog(); + } catch(Exception e){ + log.error("Error running milestone solidification thread", e); + } + } + } + + private void solidifyLog(){ + if(unsolidMilestones.size() > 1){ + if(oldestMilestoneInQueue != null){ + int milestoneIndex = oldestMilestoneInQueue.getValue(); + log.info("Solidifying milestone # " + milestoneIndex + " - [ LSM: " + latestSolidMilestone + + " LM: " + latestMilestoneIndex + " ] - [ Remaining: " + + (latestMilestoneIndex - latestSolidMilestone) + " Queued: " + + (seenMilestones.size() + unsolidMilestones.size()) + " ]"); + } + } + } /** - * Holds all unsolid milestones that shall be solidified (the transaction hash mapped to its milestone index). + * Iterates through seen milestones and ensures that they are valid. If the milestone is invalid, the index is + * checked to see if a milestone object exists already. If an existing milestone is present, then the current + * milestone index/hash pairing is replaced with the existing milestone hash. */ - private final Map unsolidMilestonesPool = new ConcurrentHashMap<>(); + private void solidifySeenMilestones() throws Exception{ + Iterator> iterator = seenMilestones.entrySet().iterator(); + Map.Entry milestone; + while(!Thread.currentThread().isInterrupted() && iterator.hasNext()){ + milestone = iterator.next(); + int milestoneIndex = milestone.getKey(); + TransactionViewModel oldMilestoneTransaction = TransactionViewModel.fromHash(tangle, milestone.getValue()); + if(milestoneService.validateMilestone(oldMilestoneTransaction, milestoneIndex) + .equals(MilestoneValidity.INVALID)){ + MilestoneViewModel nextMilestone = MilestoneViewModel.get(tangle, milestoneIndex); + if(nextMilestone != null) { + if(TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) { + seenMilestones.remove(milestoneIndex); + removeFromQueue(oldMilestoneTransaction.getHash()); + add(nextMilestone.getHash(), milestoneIndex); + } + } else { + seenMilestones.remove(milestoneIndex); + removeFromQueue(oldMilestoneTransaction.getHash()); + } + } + } + } /** - * Holds the milestones that are actively trying to be solidified by the background {@link Thread} (acts as a - * Queue). + * Iterates through valid milestones and submits them to the {@link TransactionSolidifier} */ - private final Map milestonesToSolidify = new HashMap<>(); + private void processSolidifyQueue() throws Exception { + Iterator> iterator = solidificationQueue.entrySet().iterator(); + Map.Entry milestone; + while(!Thread.currentThread().isInterrupted() && iterator.hasNext()){ + milestone = iterator.next(); + TransactionViewModel milestoneCandidate = TransactionViewModel.fromHash(tangle, milestone.getKey()); + if(milestoneService.validateMilestone(milestoneCandidate, milestone.getValue()) + .equals(MilestoneValidity.VALID)){ + milestoneCandidate.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); + if(milestoneCandidate.isSolid()) { + addSeenMilestone(milestone.getKey(), milestone.getValue()); + } else { + transactionSolidifier.addToSolidificationQueue(milestone.getKey()); + } + } else { + transactionSolidifier.addToSolidificationQueue(milestone.getKey()); + } + } + } + /** - *

- * Holds and entry that represents the youngest milestone in the {@link #milestonesToSolidify} Map. - *

- *

- * Note: It is used to check if new milestones that are being added, are older that the currently processed ones and - * should replace them in the queue (we solidify from oldest to youngest). - *

+ * Iterates through milestone indexes starting from the initial snapshot index, and returns the newest existing + * milestone index + * + * @return Latest Solid Milestone index + * @throws Exception */ - private Map.Entry youngestMilestoneInQueue = null; + private void bootStrapSolidMilestones() throws Exception { + boolean solid = true; + int milestoneIndex = snapshotProvider.getInitialSnapshot().getIndex(); + while(solid){ + milestoneIndex += 1; + if(MilestoneViewModel.get(tangle, milestoneIndex) == null){ + solid = false; + milestoneIndex -= 1; + } + } + latestSolidMilestone = milestoneIndex; + + AddressViewModel.load(tangle, config.getCoordinator()).getHashes().forEach(hash -> { + try { + int index; + if ((index = milestoneService.getMilestoneIndex(TransactionViewModel.fromHash(tangle, hash))) > + latestSolidMilestone){ + add(hash, index); + latestMilestoneIndex = index; + latestMilestoneHash = hash; + + } + } catch(Exception e) { + log.error("Error processing existing milestone index", e); + } + }); + + if(latestMilestoneIndex > snapshotProvider.getInitialSnapshot().getIndex()) { + logNewMilestone(snapshotProvider.getInitialSnapshot().getIndex(), latestMilestoneIndex, latestMilestoneHash); + } + } + /** - * @param snapshotProvider snapshot provider which gives us access to the relevant snapshots - * @param transactionValidator TransactionValidator instance that is used by the node + * Tries to solidifiy the next milestone index. If successful, the milestone will be removed from the + * {@link #seenMilestones} queue, and any milestone objects below that index in the {@link #unsolidMilestones} queue + * will be removed as well. */ - public MilestoneSolidifierImpl(SnapshotProvider snapshotProvider, TransactionValidator transactionValidator) { - this.snapshotProvider = snapshotProvider; - this.transactionValidator = transactionValidator; + private void checkLatestSolidMilestone(){ + try { + if (latestMilestoneIndex > latestSolidMilestone) { + int nextMilestone = latestSolidMilestone + 1; + if (seenMilestones.containsKey(nextMilestone)){ + TransactionViewModel milestone = TransactionViewModel.fromHash(tangle, + seenMilestones.get(nextMilestone)); + if(milestone.isSolid()){ + updateSolidMilestone(latestSolidMilestone); + transactionSolidifier.addToPropagationQueue(milestone.getHash()); + } else { + transactionSolidifier.addToSolidificationQueue(milestone.getHash()); + } + } + } + } catch (Exception e) { + log.info(e.getMessage()); + } + } + + private void checkValidity(TransactionViewModel milestone, int index) throws Exception{ + switch(milestoneService.validateMilestone(milestone, index)) { + case INVALID: + MilestoneViewModel nextMilestone = MilestoneViewModel.get(tangle, index); + if(nextMilestone != null) { + if(TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) { + seenMilestones.remove(index); + removeFromQueue(milestone.getHash()); + add(nextMilestone.getHash(), index); + } + } else { + removeFromQueue(milestone.getHash()); + } + case INCOMPLETE: + transactionSolidifier.addToSolidificationQueue(milestone.getHash()); + } + } + + private void updateSolidMilestone(int currentSolidMilestoneIndex) throws Exception{ + MilestoneViewModel nextSolidMilestone = MilestoneViewModel.get(tangle, currentSolidMilestoneIndex + 1); + if(nextSolidMilestone != null) { + applySolidMilestoneToLedger(nextSolidMilestone); + logChange(currentSolidMilestoneIndex); + + if (currentSolidMilestoneIndex + 1 == latestMilestoneIndex) { + transactionRequester.clearRecentlyRequestedTransactions(); + } + + removeSeenMilestone(currentSolidMilestoneIndex + 1); + latestSolidMilestone = currentSolidMilestoneIndex + 1; + } + } /** * {@inheritDoc} - * - *

- * Since this method might be called from a performance critical context, we simply add the milestone to a temporary - * pool, that gets examined later by the background process. This doesn't just speed up the addition of new jobs but - * also prevents us from having to synchronize the access to the underlying maps. - *

*/ @Override - public void add(Hash milestoneHash, int milestoneIndex) { - if (!unsolidMilestonesPool.containsKey(milestoneHash) && !newlyAddedMilestones.containsKey(milestoneHash) && - milestoneIndex > snapshotProvider.getInitialSnapshot().getIndex()) { + public void add(Hash milestoneHash, int milestoneIndex){ + if(!unsolidMilestones.containsKey(milestoneHash) && !seenMilestones.containsKey(milestoneIndex) && + milestoneIndex > latestSolidMilestone){ + unsolidMilestones.put(milestoneHash, milestoneIndex); + updateQueues(milestoneHash, milestoneIndex); + } + } - newlyAddedMilestones.put(milestoneHash, milestoneIndex); + private void updateQueues(Hash milestoneHash, int milestoneIndex){ + if(solidificationQueue.containsKey(milestoneHash)) { + if(solidificationQueue.size() >= MAX_SIZE){ + Iterator> iterator = solidificationQueue.entrySet().iterator(); + iterator.next(); + iterator.remove(); + } } + + solidificationQueue.put(milestoneHash, milestoneIndex); + scanMilestonesInQueue(); } + /** + * {@inheritDoc} + */ @Override - public void start() { - executorService.silentScheduleWithFixedDelay(this::milestoneSolidificationThread, 0, SOLIDIFICATION_INTERVAL, - TimeUnit.MILLISECONDS); + public void addSeenMilestone(Hash milestoneHash, int milestoneIndex){ + if(!seenMilestones.containsKey(milestoneIndex)){ + seenMilestones.put(milestoneIndex, milestoneHash); + } + removeFromQueue(milestoneHash); } + /** + * {@inheritDoc} + */ @Override - public void shutdown() { - executorService.shutdownNow(); + public List >getOldestMilestonesInQueue(){ + return new ArrayList<>(solidificationQueue.entrySet()); } /** - *

- * This method takes an entry from the {@link #unsolidMilestonesPool} and adds it to the - * {@link #milestonesToSolidify} queue. - *

- *

- * It first checks if the given milestone is already part of the queue and then tries to add it. If the queue is not - * full yet, the addition to the queue is relatively cheap, because the {@link #youngestMilestoneInQueue} marker can - * be updated without iterating over all entries. If the queue reached its capacity already, we replace the entry - * marked by the {@link #youngestMilestoneInQueue} marker and update the marker by recalculating it using - * {@link #determineYoungestMilestoneInQueue()}. - *

- * - * @param milestoneEntry entry from the {@link #unsolidMilestonesPool} that shall get added to the queue + * {@inheritDoc} */ - private void addToSolidificationQueue(Map.Entry milestoneEntry) { - if (milestonesToSolidify.containsKey(milestoneEntry.getKey())) { - return; - } - - if (milestonesToSolidify.size() < SOLIDIFICATION_QUEUE_SIZE) { - milestonesToSolidify.put(milestoneEntry.getKey(), milestoneEntry.getValue()); - - if (youngestMilestoneInQueue == null || milestoneEntry.getValue() > youngestMilestoneInQueue.getValue()) { - youngestMilestoneInQueue = milestoneEntry; - } - } else if (milestoneEntry.getValue() < youngestMilestoneInQueue.getValue()) { - milestonesToSolidify.remove(youngestMilestoneInQueue.getKey()); - milestonesToSolidify.put(milestoneEntry.getKey(), milestoneEntry.getValue()); + @Override + public void removeFromQueue(Hash milestoneHash) { + unsolidMilestones.remove(milestoneHash); + solidificationQueue.remove(milestoneHash); + } - determineYoungestMilestoneInQueue(); - } + /** + * {@inheritDoc} + */ + @Override + public int getLatestMilestoneIndex(){ + return latestMilestoneIndex; } /** - *

- * This method contains the logic for the milestone solidification, that gets executed in a separate - * {@link Thread}. - *

- *

- * It executes the necessary steps periodically while waiting a short time to give the nodes the ability to - * answer to the issued transaction requests. - *

+ * {@inheritDoc} */ - private void milestoneSolidificationThread() { - processNewlyAddedMilestones(); - processSolidificationQueue(); - refillSolidificationQueue(); + @Override + public Hash getLatestMilestoneHash(){ + return latestMilestoneHash; } /** - *

- * This method processes the newly added milestones. - *

- *

- * We process them lazy to decrease the synchronization requirements and speed up the addition of milestones from - * outside {@link Thread}s. - *

- *

- * It iterates over the milestones and adds them to the pool. If they are older than the - * {@link #youngestMilestoneInQueue}, we add the to the solidification queue. - *

+ * {@inheritDoc} */ - private void processNewlyAddedMilestones() { - for (Iterator> iterator = newlyAddedMilestones.entrySet().iterator(); - !Thread.currentThread().isInterrupted() && iterator.hasNext();) { + @Override + public void setLatestMilestone(Hash latestMilestoneHash, int latestMilestoneIndex) { + this.latestMilestoneHash = latestMilestoneHash; + this.latestMilestoneIndex = latestMilestoneIndex; + } - Map.Entry currentEntry = iterator.next(); + /** + * {@inheritDoc} + */ + @Override + public boolean isInitialScanComplete(){ + return initialized; + } - unsolidMilestonesPool.put(currentEntry.getKey(), currentEntry.getValue()); + /** + * {@inheritDoc} + */ + @Override + public void logNewMilestone(int oldMilestoneIndex, int newMilestoneIndex, Hash newMilestoneHash){ + setLatestMilestone(newMilestoneHash, newMilestoneIndex); + tangle.publish("lmi %d %d", latestMilestoneIndex, newMilestoneIndex); + log.info("Latest milestone has changed from #" + oldMilestoneIndex + " to #" + newMilestoneIndex); + } - if (youngestMilestoneInQueue == null || currentEntry.getValue() < youngestMilestoneInQueue.getValue()) { - addToSolidificationQueue(currentEntry); + /** + * Remove milestone candidate from the {@link #seenMilestones} queue, then iterate through the + * {@link #unsolidMilestones} queue and remove any milestones with an index below the provided index. + * + * @param solidMilestoneIndex The milestone index to remove + */ + private void removeSeenMilestone(int solidMilestoneIndex) { + seenMilestones.remove(solidMilestoneIndex); + for (Iterator> iterator = unsolidMilestones.entrySet().iterator(); + !Thread.currentThread().isInterrupted() && iterator.hasNext();) { + Map.Entry nextMilestone = iterator.next(); + if (nextMilestone.getValue() <= solidMilestoneIndex || + nextMilestone.getValue() < snapshotProvider.getLatestSnapshot().getInitialIndex()) { + iterator.remove(); } - - iterator.remove(); } + scanMilestonesInQueue(); } /** - *

- * This method contains the logic for processing the {@link #milestonesToSolidify}. - *

- *

- * It iterates through the queue and checks if the corresponding milestones are still relevant for our node, or if - * they could be successfully solidified. If the milestones become solid or irrelevant, we remove them from the - * pool and the queue and reset the {@link #youngestMilestoneInQueue} marker (if necessary). - *

+ * Scan through milestones in the {@link #unsolidMilestones} queue and remove any that are present in the + * {@link #seenMilestones} queue. If it isn't in the {@link #seenMilestones} queue, then the youngest and oldest + * milestone objects are updated with that milestone object. Used to reset the oldest milestones for solidification. + * If upon scanning the {@link #unsolidMilestones} queue is empty and {@link #initialized} is false, set + * {@link #initialized} to true. */ - private void processSolidificationQueue() { - for (Iterator> iterator = milestonesToSolidify.entrySet().iterator(); - !Thread.currentThread().isInterrupted() && iterator.hasNext();) { - - Map.Entry currentEntry = iterator.next(); - - if (currentEntry.getValue() <= snapshotProvider.getInitialSnapshot().getIndex() || isSolid(currentEntry)) { - unsolidMilestonesPool.remove(currentEntry.getKey()); - iterator.remove(); + private void scanMilestonesInQueue() { + if(unsolidMilestones.size() > 0) { + Map newSolidificationQueue = new ConcurrentHashMap<>(); + unsolidMilestones.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .forEach(milestone -> { + if (newSolidificationQueue.size() < MAX_SIZE) { + newSolidificationQueue.put(milestone.getKey(), milestone.getValue()); + } + }); + solidificationQueue = newSolidificationQueue; + } + oldestMilestoneInQueue = null; + for (Map.Entry currentEntry : solidificationQueue.entrySet()) { + if (seenMilestones.containsKey(currentEntry.getValue())) { + unsolidMilestones.remove(currentEntry.getKey()); + solidificationQueue.remove(currentEntry.getKey()); + } else { + updateOldestMilestone(currentEntry.getKey(), currentEntry.getValue()); + } + } + if(!initialized && unsolidMilestones.size() == 0){ + initialized = true; + } + } - if (youngestMilestoneInQueue != null && - currentEntry.getKey().equals(youngestMilestoneInQueue.getKey())) { + private void updateOldestMilestone(Hash milestoneHash, int milestoneIndex){ + if (oldestMilestoneInQueue == null || oldestMilestoneInQueue.getValue() > milestoneIndex) { + oldestMilestoneInQueue = new AbstractMap.SimpleEntry<>(milestoneHash, milestoneIndex); + } + } - youngestMilestoneInQueue = null; - } + private void applySolidMilestoneToLedger(MilestoneViewModel milestone) throws Exception { + if (ledgerService.applyMilestoneToLedger(milestone)) { + if (isRepairRunning() && isRepairSuccessful(milestone)) { + stopRepair(); } + syncProgressInfo.addMilestoneApplicationTime(); + } else { + repairCorruptedMilestone(milestone); } } /** *

- * This method takes care of adding new milestones from the pool to the solidification queue, and filling it up - * again after it was processed / emptied before. + * Tries to actively repair the ledger by reverting the milestones preceding the given milestone. + *

+ *

+ * It gets called when a milestone could not be applied to the ledger state because of problems like "inconsistent + * balances". While this should theoretically never happen (because milestones are by definition "consistent"), it + * can still happen because IRI crashed or got stopped in the middle of applying a milestone or if a milestone + * was processed in the wrong order. *

*

- * It first updates the {@link #youngestMilestoneInQueue} marker and then just adds new milestones as long as there - * is still space in the {@link #milestonesToSolidify} queue. + * Every time we call this method the internal {@link #repairBackoffCounter} is incremented which causes the next + * call of this method to repair an additional milestone. This means that whenever we face an error we first try to + * reset only the last milestone, then the two last milestones, then the three last milestones (and so on ...) until + * the problem was fixed. *

+ *

+ * To be able to tell when the problem is fixed and the {@link #repairBackoffCounter} can be reset, we store the + * milestone index that caused the problem the first time we call this method. + *

+ * + * @param errorCausingMilestone the milestone that failed to be applied + * @throws MilestoneException if we failed to reset the corrupted milestone */ - private void refillSolidificationQueue() { - if(youngestMilestoneInQueue == null && !milestonesToSolidify.isEmpty()) { - determineYoungestMilestoneInQueue(); + private void repairCorruptedMilestone(MilestoneViewModel errorCausingMilestone) throws MilestoneException { + if (repairBackoffCounter++ == 0) { + errorCausingMilestoneIndex = errorCausingMilestone.index(); } - - Map.Entry nextSolidificationCandidate; - while (!Thread.currentThread().isInterrupted() && milestonesToSolidify.size() < SOLIDIFICATION_QUEUE_SIZE && - (nextSolidificationCandidate = getNextSolidificationCandidate()) != null) { - - addToSolidificationQueue(nextSolidificationCandidate); + for (int i = errorCausingMilestone.index(); i > errorCausingMilestone.index() - repairBackoffCounter; i--) { + milestoneService.resetCorruptedMilestone(i); } } /** *

- * This method determines the youngest milestone in the solidification queue. + * Checks if we are currently trying to repair a milestone. *

*

- * It iterates over all milestones in the Queue and keeps track of the youngest one found (the one with the highest - * milestone index). + * We simply use the {@link #repairBackoffCounter} as an indicator if a repair routine is running. *

+ * + * @return {@code true} if we are trying to repair a milestone and {@code false} otherwise */ - private void determineYoungestMilestoneInQueue() { - youngestMilestoneInQueue = null; - for (Map.Entry currentEntry : milestonesToSolidify.entrySet()) { - if (youngestMilestoneInQueue == null || currentEntry.getValue() > youngestMilestoneInQueue.getValue()) { - youngestMilestoneInQueue = currentEntry; - } - } + private boolean isRepairRunning() { + return repairBackoffCounter != 0; } /** *

- * This method returns the earliest seen Milestone from the unsolid milestones pool, that is not part of the - * {@link #milestonesToSolidify} queue yet. + * Checks if we successfully repaired the corrupted milestone. *

*

- * It simply iterates over all milestones in the pool and looks for the one with the lowest index, that is not - * getting actively solidified, yet. + * To determine if the repair routine was successful we check if the processed milestone has a higher index than the + * one that initially could not get applied to the ledger. *

* - * @return the Map.Entry holding the earliest milestone or null if the pool does not contain any new candidates. + * @param processedMilestone the currently processed milestone + * @return {@code true} if we advanced to a milestone following the corrupted one and {@code false} otherwise */ - private Map.Entry getNextSolidificationCandidate() { - Map.Entry nextSolidificationCandidate = null; - for (Map.Entry milestoneEntry : unsolidMilestonesPool.entrySet()) { - if (!milestonesToSolidify.containsKey(milestoneEntry.getKey()) && (nextSolidificationCandidate == null || - milestoneEntry.getValue() < nextSolidificationCandidate.getValue())) { - - nextSolidificationCandidate = milestoneEntry; - } - } - - return nextSolidificationCandidate; + private boolean isRepairSuccessful(MilestoneViewModel processedMilestone) { + return processedMilestone.index() > errorCausingMilestoneIndex; } /** *

- * This method performs the actual solidity check on the selected milestone. - *

- *

- * It first dumps a log message to keep the node operator informed about the progress of solidification, and then - * issues the {@link TransactionValidator#checkSolidity(Hash, int)} call that starts the solidification - * process. + * Resets the internal variables that are used to keep track of the repair process. *

*

- * We limit the amount of transactions that may be processed during the solidity check, since we want to solidify - * from the oldest milestone to the newest one and not "block" the solidification with a very recent milestone that - * needs to traverse huge chunks of the tangle. The main goal of this is to give the solidification just enough - * "resources" to discover the previous milestone while at the same time allowing fast solidity checks. + * It gets called whenever we advance to a milestone that has a higher milestone index than the milestone that + * initially caused the repair routine to kick in (see {@link #repairCorruptedMilestone(MilestoneViewModel)}. *

- * - * @param currentEntry milestone entry that shall be checked - * @return true if the given milestone is solid or false otherwise */ - private boolean isSolid(Map.Entry currentEntry) { - if (unsolidMilestonesPool.size() > 1) { - log.info("Solidifying milestone #" + currentEntry.getValue() + - " [" + milestonesToSolidify.size() + " / " + unsolidMilestonesPool.size() + "]"); + private void stopRepair() { + repairBackoffCounter = 0; + errorCausingMilestoneIndex = Integer.MAX_VALUE; + } + + + private void logChange(int prevSolidMilestoneIndex) { + Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); + int latestSolidMilestoneIndex = latestSnapshot.getIndex(); + + if (prevSolidMilestoneIndex == latestSolidMilestoneIndex) { + return; } - try { - return transactionValidator.checkSolidity(currentEntry.getKey(), SOLIDIFICATION_TRANSACTIONS_LIMIT); - } catch (Exception e) { - log.error("Error while solidifying milestone #" + currentEntry.getValue(), e); + Hash latestMilestoneHash = latestSnapshot.getHash(); + log.delegate().info("Latest SOLID milestone index changed from #" + prevSolidMilestoneIndex + " to #" + latestSolidMilestoneIndex); + + tangle.publish("lmsi %d %d", prevSolidMilestoneIndex, latestSolidMilestoneIndex); + tangle.publish("lmhs %s", latestMilestoneHash); - return false; + if (!config.isPrintSyncProgressEnabled()) { + return; + } + + int latestMilestoneIndex = getLatestMilestoneIndex(); + + // only print more sophisticated progress if we are coming from a more unsynced state + if(latestMilestoneIndex - latestSolidMilestoneIndex < 1) { + syncProgressInfo.setSyncMilestoneStartIndex(latestSolidMilestoneIndex); + syncProgressInfo.resetMilestoneApplicationTimes(); + return; + } + + int estSecondsToBeSynced = syncProgressInfo.computeEstimatedTimeToSyncUpSeconds(latestMilestoneIndex, + latestSolidMilestoneIndex); + StringBuilder progressSB = new StringBuilder(); + + // add progress bar + progressSB.append(ASCIIProgressBar.getProgressBarString(syncProgressInfo.getSyncMilestoneStartIndex(), + latestMilestoneIndex, latestSolidMilestoneIndex)); + // add lsm to lm + progressSB.append(String.format(" [LSM %d / LM %d - remaining: %d]", latestSolidMilestoneIndex, + latestMilestoneIndex, latestMilestoneIndex - latestSolidMilestoneIndex)); + // add estimated time to get fully synced + if (estSecondsToBeSynced != -1) { + progressSB.append(String.format(" - est. seconds to get synced: %d", estSecondsToBeSynced)); + } + log.info(progressSB.toString()); + } + + + /** + * Holds variables containing information needed for sync progress calculation. + */ + private static class SyncProgressInfo { + /** + * The actual start milestone index from which the node started from when syncing up. + */ + private int syncMilestoneStartIndex; + + /** + * Used to calculate the average time needed to apply a milestone. + */ + private BlockingQueue lastMilestoneApplyTimes = new ArrayBlockingQueue<>(100); + + /** + * Gets the milestone sync start index. + * + * @return the milestone sync start index + */ + int getSyncMilestoneStartIndex() { + return syncMilestoneStartIndex; + } + + /** + * Sets the milestone sync start index. + * + * @param syncMilestoneStartIndex the value to set + */ + void setSyncMilestoneStartIndex(int syncMilestoneStartIndex) { + this.syncMilestoneStartIndex = syncMilestoneStartIndex; + } + + /** + * Adds the current time as a time where a milestone got applied to the ledger state. + */ + void addMilestoneApplicationTime(){ + if(lastMilestoneApplyTimes.remainingCapacity() == 0){ + lastMilestoneApplyTimes.remove(); + } + lastMilestoneApplyTimes.add(System.currentTimeMillis()); + } + + /** + * Clears the records of times when milestones got applied. + */ + void resetMilestoneApplicationTimes(){ + lastMilestoneApplyTimes.clear(); + } + + /** + * Computes the estimated time (seconds) needed to get synced up by looking at the last N milestone applications + * times and the current solid and last milestone indices. + * + * @param latestMilestoneIndex the current latest known milestone index + * @param latestSolidMilestoneIndex the current to the ledger applied milestone index + * @return the number of seconds needed to get synced up or -1 if not enough data is available + */ + int computeEstimatedTimeToSyncUpSeconds(int latestMilestoneIndex, int latestSolidMilestoneIndex) { + // compute average time needed to apply a milestone + Object[] times = lastMilestoneApplyTimes.toArray(); + long sumDelta = 0; + double avgMilestoneApplyMillisec; + if (times.length > 1) { + // compute delta sum + for (int i = times.length - 1; i > 1; i--) { + sumDelta += ((Long)times[i]) - ((Long)times[i - 1]); + } + avgMilestoneApplyMillisec = (double) sumDelta / (double) (times.length - 1); + } else { + return -1; + } + + return (int) ((avgMilestoneApplyMillisec / 1000) * (latestMilestoneIndex - latestSolidMilestoneIndex)); } } + } diff --git a/src/main/java/com/iota/iri/service/snapshot/LocalSnapshotManager.java b/src/main/java/com/iota/iri/service/snapshot/LocalSnapshotManager.java index 790113ce78..b78b8b08e9 100644 --- a/src/main/java/com/iota/iri/service/snapshot/LocalSnapshotManager.java +++ b/src/main/java/com/iota/iri/service/snapshot/LocalSnapshotManager.java @@ -1,6 +1,6 @@ package com.iota.iri.service.snapshot; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.transactionpruning.PruningCondition; /** @@ -18,13 +18,13 @@ public interface LocalSnapshotManager { * Note: If the node is not fully synced we use * {@link com.iota.iri.conf.SnapshotConfig#getLocalSnapshotsIntervalUnsynced()} instead. * - * @param latestMilestoneTracker tracker for the milestones to determine when a new local snapshot is due + * @param milestoneSolidifier tracker for the milestones to determine when a new local snapshot is due */ - void start(LatestMilestoneTracker latestMilestoneTracker); + void start(MilestoneSolidifier milestoneSolidifier); /** * Stops the {@link Thread} that takes care of creating the local {@link Snapshot}s and that was spawned by the - * {@link #start(LatestMilestoneTracker)} method. + * {@link #start(MilestoneSolidifier)} method. */ void shutdown(); diff --git a/src/main/java/com/iota/iri/service/snapshot/SnapshotService.java b/src/main/java/com/iota/iri/service/snapshot/SnapshotService.java index 52b64e8c75..687c89d97d 100644 --- a/src/main/java/com/iota/iri/service/snapshot/SnapshotService.java +++ b/src/main/java/com/iota/iri/service/snapshot/SnapshotService.java @@ -4,7 +4,7 @@ import com.iota.iri.controllers.MilestoneViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.transactionpruning.TransactionPruner; /** @@ -71,16 +71,15 @@ public interface SnapshotService { * by the {@code snapshotProvider} to reflect the newly created {@link Snapshot}. *

* - * @param latestMilestoneTracker milestone tracker that allows us to retrieve information about the known milestones + * @param milestoneSolidifier milestone tracker that allows us to retrieve information about the known milestones * @param transactionPruner manager for the pruning jobs that takes care of cleaning up the old data that * @param snapshotUntillIndex The last milestone we keep, everything before gets snapshotted. * If we can't find the milestone of this index, we attempt to look back further until we do * @return The Snapshot we ended up making * @throws SnapshotException if anything goes wrong while creating the local snapshot */ - Snapshot takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, TransactionPruner transactionPruner, - int snapshotUntillIndex) throws - SnapshotException; + Snapshot takeLocalSnapshot(MilestoneSolidifier milestoneSolidifier, TransactionPruner transactionPruner, + int snapshotUntillIndex) throws SnapshotException; /** *

@@ -106,12 +105,12 @@ void pruneSnapshotData(TransactionPruner transactionPruner, int pruningMilestone * local snapshot files. *

* - * @param latestMilestoneTracker milestone tracker that allows us to retrieve information about the known milestones + * @param milestoneSolidifier milestone tracker that allows us to retrieve information about the known milestones * @param targetMilestone milestone that is used as a reference point for the snapshot * @return a local snapshot of the full ledger state at the given milestone * @throws SnapshotException if anything goes wrong while generating the local snapshot */ - Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, MilestoneViewModel targetMilestone) throws + Snapshot generateSnapshot(MilestoneSolidifier milestoneSolidifier, MilestoneViewModel targetMilestone) throws SnapshotException; /** @@ -140,11 +139,11 @@ Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, Milesto * very first time. *

* - * @param latestMilestoneTracker milestone tracker that allows us to retrieve information about the known milestones + * @param milestoneSolidifier milestone tracker that allows us to retrieve information about the known milestones * @param targetMilestone milestone that is used as a reference point for the snapshot * @return a map of solid entry points associating their hash to the milestone index that confirmed them * @throws SnapshotException if anything goes wrong while generating the solid entry points */ - Map generateSeenMilestones(LatestMilestoneTracker latestMilestoneTracker, - MilestoneViewModel targetMilestone) throws SnapshotException; + Map generateSeenMilestones(MilestoneSolidifier milestoneSolidifier, + MilestoneViewModel targetMilestone) throws SnapshotException; } diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImpl.java index 95429cf138..06c01344d9 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImpl.java @@ -2,7 +2,7 @@ import com.iota.iri.conf.BaseIotaConfig; import com.iota.iri.conf.SnapshotConfig; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.LocalSnapshotManager; import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.snapshot.SnapshotCondition; @@ -28,7 +28,7 @@ *

*

* It incorporates a background worker that periodically checks if a new snapshot is due (see {@link - * #start(LatestMilestoneTracker)} and {@link #shutdown()}). + * #start(MilestoneSolidifier)} and {@link #shutdown()}). *

*/ public class LocalSnapshotManagerImpl implements LocalSnapshotManager { @@ -79,7 +79,7 @@ public class LocalSnapshotManagerImpl implements LocalSnapshotManager { * Holds a reference to the {@link ThreadIdentifier} for the monitor thread. * * Using a {@link ThreadIdentifier} for spawning the thread allows the {@link ThreadUtils} to spawn exactly one - * thread for this instance even when we call the {@link #start(LatestMilestoneTracker)} method multiple times. + * thread for this instance even when we call the {@link #start(MilestoneSolidifier)} method multiple times. */ private ThreadIdentifier monitorThreadIdentifier = new ThreadIdentifier("Local Snapshots Monitor"); @@ -105,8 +105,8 @@ public LocalSnapshotManagerImpl(SnapshotProvider snapshotProvider, SnapshotServi * {@inheritDoc} */ @Override - public void start(LatestMilestoneTracker latestMilestoneTracker) { - ThreadUtils.spawnThread(() -> monitorThread(latestMilestoneTracker), monitorThreadIdentifier); + public void start(MilestoneSolidifier milestoneSolidifier) { + ThreadUtils.spawnThread(() -> monitorThread(milestoneSolidifier), monitorThreadIdentifier); } /** @@ -123,15 +123,15 @@ public void shutdown() { * It periodically checks if a new {@link com.iota.iri.service.snapshot.Snapshot} has to be taken until the * {@link Thread} is terminated. If it detects that a {@link com.iota.iri.service.snapshot.Snapshot} is due it * triggers the creation of the {@link com.iota.iri.service.snapshot.Snapshot} by calling - * {@link SnapshotService#takeLocalSnapshot}. + * {@link SnapshotService#takeLocalSnapshot(MilestoneSolidifier, TransactionPruner, int)}. * - * @param latestMilestoneTracker tracker for the milestones to determine when a new local snapshot is due + * @param milestoneSolidifier tracker for the milestones to determine when a new local snapshot is due */ @VisibleForTesting - void monitorThread(LatestMilestoneTracker latestMilestoneTracker) { + void monitorThread(MilestoneSolidifier milestoneSolidifier) { while (!Thread.currentThread().isInterrupted()) { // Possibly takes a snapshot if we can - handleSnapshot(latestMilestoneTracker); + handleSnapshot(milestoneSolidifier); // Prunes data separate of a snapshot if we made a snapshot of the pruned data now or previously if (config.getLocalSnapshotsPruningEnabled()) { @@ -141,14 +141,14 @@ void monitorThread(LatestMilestoneTracker latestMilestoneTracker) { } } - private Snapshot handleSnapshot(LatestMilestoneTracker latestMilestoneTracker) { - boolean isInSync = isInSync(latestMilestoneTracker); + private Snapshot handleSnapshot(MilestoneSolidifier milestoneSolidifier) { + boolean isInSync = isInSync(milestoneSolidifier); int lowestSnapshotIndex = calculateLowestSnapshotIndex(isInSync); - if (canTakeSnapshot(lowestSnapshotIndex, latestMilestoneTracker)) { + if (canTakeSnapshot(lowestSnapshotIndex, milestoneSolidifier)) { try { log.debug("Taking snapshot at index {}", lowestSnapshotIndex); return snapshotService.takeLocalSnapshot( - latestMilestoneTracker, transactionPruner, lowestSnapshotIndex); + milestoneSolidifier, transactionPruner, lowestSnapshotIndex); } catch (SnapshotException e) { log.error("error while taking local snapshot", e); } catch (Exception e) { @@ -180,9 +180,9 @@ private int calculateLowestSnapshotIndex(boolean isInSync) { return lowestSnapshotIndex; } - private boolean canTakeSnapshot(int lowestSnapshotIndex, LatestMilestoneTracker latestMilestoneTracker) { + private boolean canTakeSnapshot(int lowestSnapshotIndex, MilestoneSolidifier milestoneSolidifier) { return lowestSnapshotIndex != -1 - && latestMilestoneTracker.isInitialScanComplete() + && milestoneSolidifier.isInitialScanComplete() && lowestSnapshotIndex > snapshotProvider.getInitialSnapshot().getIndex() && lowestSnapshotIndex <= snapshotProvider.getLatestSnapshot().getIndex() - config.getLocalSnapshotsDepth(); } @@ -252,16 +252,16 @@ private boolean canPrune(int pruningMilestoneIndex) { * This will always return false if we are not done scanning milestone * candidates during initialization. * - * @param latestMilestoneTracker tracker we use to determine milestones + * @param milestoneSolidifier tracker we use to determine milestones * @return true if we are in sync, otherwise false */ @VisibleForTesting - boolean isInSync(LatestMilestoneTracker latestMilestoneTracker) { - if (!latestMilestoneTracker.isInitialScanComplete()) { + boolean isInSync(MilestoneSolidifier milestoneSolidifier) { + if (!milestoneSolidifier.isInitialScanComplete()) { return false; } - int latestIndex = latestMilestoneTracker.getLatestMilestoneIndex(); + int latestIndex = milestoneSolidifier.getLatestMilestoneIndex(); int latestSnapshot = snapshotProvider.getLatestSnapshot().getIndex(); // If we are out of sync, only a full sync will get us in diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java index 02f0a5bb06..b85c2ad047 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java @@ -6,7 +6,7 @@ import com.iota.iri.controllers.StateDiffViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.snapshot.SnapshotException; import com.iota.iri.service.snapshot.SnapshotMetaData; @@ -200,12 +200,12 @@ public void rollBackMilestones(Snapshot snapshot, int targetMilestoneIndex) thro * {@inheritDoc} */ @Override - public Snapshot takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, TransactionPruner transactionPruner, int targetMilestoneIndex) - throws SnapshotException { + public Snapshot takeLocalSnapshot(MilestoneSolidifier milestoneSolidifier, TransactionPruner transactionPruner, + int snapshotUntillIndex) throws SnapshotException { - MilestoneViewModel targetMilestone = determineMilestoneForLocalSnapshot(tangle, snapshotProvider, targetMilestoneIndex); - - Snapshot newSnapshot = generateSnapshot(latestMilestoneTracker, targetMilestone); + MilestoneViewModel targetMilestone = determineMilestoneForLocalSnapshot(tangle, snapshotProvider, + snapshotUntillIndex); + Snapshot newSnapshot = generateSnapshot(milestoneSolidifier, targetMilestone); if (transactionPruner != null) { cleanupExpiredSolidEntryPoints(tangle, snapshotProvider.getInitialSnapshot().getSolidEntryPoints(), @@ -230,7 +230,7 @@ public void pruneSnapshotData(TransactionPruner transactionPruner, int pruningMi * {@inheritDoc} */ @Override - public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, MilestoneViewModel targetMilestone) + public Snapshot generateSnapshot(MilestoneSolidifier milestoneSolidifier, MilestoneViewModel targetMilestone) throws SnapshotException { if (targetMilestone == null) { @@ -266,7 +266,7 @@ public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, } snapshot.setSolidEntryPoints(generateSolidEntryPoints(targetMilestone)); - snapshot.setSeenMilestones(generateSeenMilestones(latestMilestoneTracker, targetMilestone)); + snapshot.setSeenMilestones(generateSeenMilestones(milestoneSolidifier, targetMilestone)); return snapshot; } @@ -289,8 +289,8 @@ public Map generateSolidEntryPoints(MilestoneViewModel targetMile * {@inheritDoc} */ @Override - public Map generateSeenMilestones(LatestMilestoneTracker latestMilestoneTracker, - MilestoneViewModel targetMilestone) throws SnapshotException { + public Map generateSeenMilestones(MilestoneSolidifier milestoneSolidifier, + MilestoneViewModel targetMilestone) throws SnapshotException { ProgressLogger progressLogger = new IntervalProgressLogger( "Taking local snapshot [processing seen milestones]", log) @@ -300,7 +300,7 @@ public Map generateSeenMilestones(LatestMilestoneTracker latestMi try { MilestoneViewModel seenMilestone = targetMilestone; while ((seenMilestone = MilestoneViewModel.findClosestNextMilestone(tangle, seenMilestone.index(), - latestMilestoneTracker.getLatestMilestoneIndex())) != null) { + milestoneSolidifier.getLatestMilestoneIndex())) != null) { seenMilestones.put(seenMilestone.getHash(), seenMilestone.index()); diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java index 8e80247c2a..bfa9d0a332 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java @@ -2,7 +2,7 @@ import com.iota.iri.controllers.MilestoneViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.tipselection.EntryPointSelector; import com.iota.iri.storage.Tangle; @@ -16,19 +16,19 @@ public class EntryPointSelectorImpl implements EntryPointSelector { private final Tangle tangle; private final SnapshotProvider snapshotProvider; - private final LatestMilestoneTracker latestMilestoneTracker; + private final MilestoneSolidifier milestoneSolidifier; /** * Constructor for Entry Point Selector * @param tangle Tangle object which acts as a database interface. * @param snapshotProvider accesses snapshots of the ledger state - * @param latestMilestoneTracker used to get latest milestone. + * @param milestoneSolidifier used to get latest milestone. */ public EntryPointSelectorImpl(Tangle tangle, SnapshotProvider snapshotProvider, - LatestMilestoneTracker latestMilestoneTracker) { + MilestoneSolidifier milestoneSolidifier) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; - this.latestMilestoneTracker = latestMilestoneTracker; + this.milestoneSolidifier = milestoneSolidifier; } @Override @@ -36,7 +36,7 @@ public Hash getEntryPoint(int depth) throws Exception { int milestoneIndex = Math.max(snapshotProvider.getLatestSnapshot().getIndex() - depth - 1, snapshotProvider.getInitialSnapshot().getIndex()); MilestoneViewModel milestoneViewModel = MilestoneViewModel.findClosestNextMilestone(tangle, milestoneIndex, - latestMilestoneTracker.getLatestMilestoneIndex()); + milestoneSolidifier.getLatestMilestoneIndex()); if (milestoneViewModel != null && milestoneViewModel.getHash() != null) { return milestoneViewModel.getHash(); } diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java new file mode 100644 index 0000000000..3e2ed5bec5 --- /dev/null +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -0,0 +1,126 @@ +package com.iota.iri.service.validation; + +import com.iota.iri.controllers.TipsViewModel; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.service.validation.impl.TransactionSolidifierImpl; +import com.iota.iri.network.TransactionRequester; + +import java.util.Set; + +/** + * Solidification tool. Transactions placed into the solidification queue will be checked for solidity. Any missing + * reference transactions will be placed into the {@link TransactionRequester}. If a transaction is found to be solid + * it is updated as such and placed into the BroadcastQueue to be sent off to the node's neighbours. + */ +public interface TransactionSolidifier { + + /** + * Initialize the executor service. Start processing transactions to solidify. + */ + void start(); + + /** + * Interrupt thread processes and shut down the executor service. + */ + void shutdown(); + + /** + * Add a hash to the solidification queue, and runs an initial {@link #checkSolidity} call. + * + * @param hash Hash of the transaction to solidify + */ + void addToSolidificationQueue(Hash hash); + + /** + * Checks if milestone transaction is solid. Returns true if it is, and if it is not, it adds the hash to the + * solidification queue and returns false. + * + * @param hash Hash of the transaction to solidify + * @param maxToProcess Maximum number of transactions to analyze + * @return True if solid, false if not + */ + boolean addMilestoneToSolidificationQueue(Hash hash, int maxToProcess); + /** + * Fetch a copy of the current transactionsToBroadcast set. + * @return A set of {@link TransactionViewModel} objects to be broadcast. + */ + Set getBroadcastQueue(); + + /** + * Remove any broadcasted transactions from the transactionsToBroadcast set + * @param transactionsBroadcasted A set of {@link TransactionViewModel} objects to remove from the set. + */ + void clearFromBroadcastQueue(Set transactionsBroadcasted); + + /** + * This method does the same as {@link #checkSolidity(Hash, int)} but defaults to an unlimited amount + * of transactions that are allowed to be traversed. + * + * @param hash hash of the transactions that shall get checked + * @return true if the transaction is solid and false otherwise + * @throws Exception if anything goes wrong while trying to solidify the transaction + */ + boolean checkSolidity(Hash hash) throws Exception; + + /** + * This method checks transactions for solidity and marks them accordingly if they are found to be solid. + * + * It iterates through all approved transactions until it finds one that is missing in the database or until it + * reached solid transactions on all traversed subtangles. In case of a missing transactions it issues a transaction + * request and returns false. If no missing transaction is found, it marks the processed transactions as solid in + * the database and returns true. + * + * Since this operation can potentially take a long time to terminate if it would have to traverse big parts of the + * tangle, it is possible to limit the amount of transactions that are allowed to be processed, while looking for + * unsolid / missing approvees. This can be useful when trying to "interrupt" the solidification of one transaction + * (if it takes too many steps) to give another one the chance to be solidified instead (i.e. prevent blocks in the + * solidification threads). + * + * @param hash hash of the transactions that shall get checked + * @param maxProcessedTransactions the maximum amount of transactions that are allowed to be traversed + * @return true if the transaction is solid and false otherwise + * @throws Exception if anything goes wrong while trying to solidify the transaction + */ + boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception; + + /** + * Updates a transaction after it was stored in the tangle. Tells the node to not request the transaction anymore, + * to update the live tips accordingly, and attempts to quickly solidify the transaction. + * + *

+ * Performs the following operations: + * + *

    + *
  1. Removes {@code transactionViewModel}'s hash from the the request queue since we already found it.
    2. + *
  2. If {@code transactionViewModel} has no children (approvers), we add it to the node's active tip list.
    3. + *
  3. Removes {@code transactionViewModel}'s parents (branch & trunk) from the node's tip list + * (if they're present there).
    4. + *
  4. Attempts to quickly solidify {@code transactionViewModel} by checking whether its direct parents + * are solid. If solid we add it to the queue transaction solidification thread to help it propagate the + * solidification to the approving child transactions.
    5. + *
  5. Requests missing direct parent (trunk & branch) transactions that are needed to solidify + * {@code transactionViewModel}.
    6. + *
+ * @param transactionViewModel received transaction that is being updated + * @throws Exception if an error occurred while trying to solidify + * @see TipsViewModel + */ + void updateStatus(TransactionViewModel transactionViewModel) throws Exception; + + /** + * Tries to solidify the transactions quickly by performing {@link TransactionSolidifierImpl#checkApproovee} on + * both parents (trunk and branch). If the parents are solid, mark the transactions as solid. + * @param transactionViewModel transaction to solidify + * @return true if we made the transaction solid, else false. + * @throws Exception + */ + boolean quickSetSolid(TransactionViewModel transactionViewModel) throws Exception; + + /** + * Add to the propagation queue where it will be processed to help solidify approving transactions faster + * @param hash The transaction hash to be removed + * @throws Exception + */ + void addToPropagationQueue(Hash hash) throws Exception; +} diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java new file mode 100644 index 0000000000..f2177b585f --- /dev/null +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -0,0 +1,164 @@ +package com.iota.iri.service.validation; + +import com.google.common.annotations.VisibleForTesting; +import com.iota.iri.conf.ProtocolConfig; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.crypto.Curl; +import com.iota.iri.crypto.Sponge; +import com.iota.iri.crypto.SpongeFactory; +import com.iota.iri.model.TransactionHash; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.snapshot.SnapshotProvider; + +import static com.iota.iri.controllers.TransactionViewModel.*; + +public class TransactionValidator { + private static final int TESTNET_MWM_CAP = 13; + + private final SnapshotProvider snapshotProvider; + private final TransactionRequester transactionRequester; + private int minWeightMagnitude = 81; + private static final long MAX_TIMESTAMP_FUTURE = 2L * 60L * 60L; + private static final long MAX_TIMESTAMP_FUTURE_MS = MAX_TIMESTAMP_FUTURE * 1_000L; + + + /** + * Constructor for Tangle Validator + * + * @param snapshotProvider data provider for the snapshots that are relevant for the node + * @param transactionRequester used to request missing transactions from neighbors + * @param protocolConfig used for checking if we are in testnet and mwm. testnet true if we are in testnet + * mode, this caps {@code mwm} to {@value #TESTNET_MWM_CAP} regardless of parameter input. + * minimum weight magnitude: the minimal number of 9s that ought to appear at the end of the + * transaction hash + */ + public TransactionValidator(SnapshotProvider snapshotProvider, TransactionRequester transactionRequester, ProtocolConfig protocolConfig) { + this.snapshotProvider = snapshotProvider; + this.transactionRequester = transactionRequester; + setMwm(protocolConfig.isTestnet(), protocolConfig.getMwm()); + } + + /** + * Set the Minimum Weight Magnitude for validation checks. + */ + @VisibleForTesting + void setMwm(boolean testnet, int mwm) { + minWeightMagnitude = mwm; + + //lowest allowed MWM encoded in 46 bytes. + if (!testnet){ + minWeightMagnitude = Math.max(minWeightMagnitude, TESTNET_MWM_CAP); + } + } + + /** + * @return the minimal number of trailing 9s that have to be present at the end of the transaction hash + * in order to validate that sufficient proof of work has been done + */ + public int getMinWeightMagnitude() { + return minWeightMagnitude; + } + + /** + * Checks that the timestamp of the transaction is below the last global snapshot time + * or more than {@value #MAX_TIMESTAMP_FUTURE} seconds in the future, and thus invalid. + * + *

+ * First the attachment timestamp (set after performing POW) is checked, and if not available + * the regular timestamp is checked. Genesis transaction will always be valid. + *

+ * @param transactionViewModel transaction under test + * @return true if timestamp is not in valid bounds and {@code transactionViewModel} is not genesis. + * Else returns false. + */ + private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) { + // ignore invalid timestamps for transactions that were requested by our node while solidifying a milestone + if(transactionRequester.wasTransactionRecentlyRequested(transactionViewModel.getHash())) { + return false; + } + + if (transactionViewModel.getAttachmentTimestamp() == 0) { + return transactionViewModel.getTimestamp() < snapshotProvider.getInitialSnapshot().getTimestamp() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(transactionViewModel.getHash()) + || transactionViewModel.getTimestamp() > (System.currentTimeMillis() / 1000) + MAX_TIMESTAMP_FUTURE; + } + return transactionViewModel.getAttachmentTimestamp() < (snapshotProvider.getInitialSnapshot().getTimestamp() * 1000L) + || transactionViewModel.getAttachmentTimestamp() > System.currentTimeMillis() + MAX_TIMESTAMP_FUTURE_MS; + } + + /** + * Runs the following validation checks on a transaction: + *
    + *
  1. {@link #hasInvalidTimestamp} check.
    2. + *
  2. Check that no value trits are set beyond the usable index, otherwise we will have values larger + * than max supply.
    3. + *
  3. Check that sufficient POW was performed.
    4. + *
  4. In value transactions, we check that the address has 0 set as the last trit. This must be because of the + * conversion between bytes to trits.
    5. + *
+ *Exception is thrown upon failure. + * + * @param transactionViewModel transaction that should be validated + * @param minWeightMagnitude the minimal number of trailing 9s at the end of the transaction hash + * @throws StaleTimestampException if timestamp check fails + * @throws IllegalStateException if any of the other checks fail + */ + public void runValidation(TransactionViewModel transactionViewModel, final int minWeightMagnitude) { + transactionViewModel.setMetadata(); + transactionViewModel.setAttachmentData(); + if(hasInvalidTimestamp(transactionViewModel)) { + throw new StaleTimestampException("Invalid transaction timestamp."); + } + for (int i = VALUE_TRINARY_OFFSET + VALUE_USABLE_TRINARY_SIZE; i < VALUE_TRINARY_OFFSET + VALUE_TRINARY_SIZE; i++) { + if (transactionViewModel.trits()[i] != 0) { + throw new IllegalStateException("Invalid transaction value"); + } + } + + int weightMagnitude = transactionViewModel.weightMagnitude; + if(weightMagnitude < minWeightMagnitude) { + throw new IllegalStateException("Invalid transaction hash"); + } + + if (transactionViewModel.value() != 0 && transactionViewModel.getAddressHash().trits()[Curl.HASH_LENGTH - 1] != 0) { + throw new IllegalStateException("Invalid transaction address"); + } + } + + /** + * Creates a new transaction from {@code trits} and validates it with {@link #runValidation}. + * + * @param trits raw transaction trits + * @param minWeightMagnitude minimal number of trailing 9s in transaction for POW validation + * @return the transaction resulting from the raw trits if valid. + * @throws RuntimeException if validation fails + */ + public TransactionViewModel validateTrits(final byte[] trits, int minWeightMagnitude) { + TransactionViewModel transactionViewModel = new TransactionViewModel(trits, TransactionHash.calculate(trits, 0, trits.length, SpongeFactory.create(SpongeFactory.Mode.CURLP81))); + runValidation(transactionViewModel, minWeightMagnitude); + return transactionViewModel; + } + + /** + * Creates a new transaction from {@code bytes} and validates it with {@link #runValidation}. + * + * @param bytes raw transaction bytes + * @param minWeightMagnitude minimal number of trailing 9s in transaction for POW validation + * @return the transaction resulting from the raw bytes if valid + * @throws RuntimeException if validation fails + */ + public TransactionViewModel validateBytes(final byte[] bytes, int minWeightMagnitude, Sponge curl) { + TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, TransactionHash.calculate(bytes, TRINARY_SIZE, curl)); + runValidation(transactionViewModel, minWeightMagnitude); + return transactionViewModel; + } + + /** + * Thrown if transaction fails {@link #hasInvalidTimestamp} check. + */ + public static class StaleTimestampException extends RuntimeException { + StaleTimestampException (String message) { + super(message); + } + } + +} diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java new file mode 100644 index 0000000000..f040f94a57 --- /dev/null +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -0,0 +1,389 @@ +package com.iota.iri.service.validation.impl; + +import com.google.common.annotations.VisibleForTesting; +import com.iota.iri.controllers.TipsViewModel; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.network.pipeline.TransactionProcessingPipeline; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.storage.Tangle; +import com.iota.iri.utils.log.interval.IntervalLogger; +import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; +import com.iota.iri.utils.thread.SilentScheduledExecutorService; + +import java.util.*; +import java.util.concurrent.*; + +import static com.iota.iri.controllers.TransactionViewModel.PREFILLED_SLOT; +import static com.iota.iri.controllers.TransactionViewModel.fromHash; + +/** + * A solidifier class for processing transactions. Transactions are checked for solidity, and missing transactions are + * subsequently requested. Once a transaction is solidified correctly it is placed into a broadcasting set to be sent to + * neighboring nodes. + */ +public class TransactionSolidifierImpl implements TransactionSolidifier { + + private Tangle tangle; + private SnapshotProvider snapshotProvider; + private TransactionRequester transactionRequester; + + /** + * Max size for all queues. + */ + private static final int MAX_SIZE= 100; + + private static final IntervalLogger log = new IntervalLogger(TransactionSolidifier.class); + + /** + * Executor service for running the {@link #processTransactionsToSolidify()}. + */ + private SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( + "Transaction Solidifier", log.delegate()); + /** + * Interval that the {@link #processTransactionsToSolidify()} will scan at. + */ + private static final int RESCAN_INTERVAL = 50; + /** + * A queue for processing transactions with the {@link #checkSolidity(Hash)} call. Once a transaction has been + * marked solid it will be placed into the {@link #transactionsToBroadcast} queue. + */ + private BlockingQueue transactionsToSolidify = new ArrayBlockingQueue<>(MAX_SIZE); + + private BlockingQueue solidTransactions = new ArrayBlockingQueue<>(MAX_SIZE); + + /** + * A set of transactions that will be called by the {@link TransactionProcessingPipeline} to be broadcast to + * neighboring nodes. + */ + private BlockingQueue transactionsToBroadcast = new ArrayBlockingQueue<>(MAX_SIZE); + + private TipsViewModel tipsViewModel; + + private Hash cooAddress; + + /** + * Constructor for the solidifier. + * @param tangle The DB reference + * @param snapshotProvider For fetching entry points for solidity checks + * @param transactionRequester A requester for missing transactions + */ + public TransactionSolidifierImpl(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester, + TipsViewModel tipsViewModel, Hash cooAddress){ + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + this.transactionRequester = transactionRequester; + this.tipsViewModel = tipsViewModel; + this.cooAddress = cooAddress; + } + + /** + *{@inheritDoc} + */ + @Override + public void start(){ + executorService.silentExecute(this::processTransactionsToSolidify); + + } + + /** + *{@inheritDoc} + */ + @Override + public void shutdown() { + executorService.shutdownNow(); + } + + /** + *{@inheritDoc} + */ + @Override + public void addToSolidificationQueue(Hash hash){ + try{ + if(!transactionsToSolidify.contains(hash)) { + + if(transactionsToSolidify.size() >= MAX_SIZE - 1){ + transactionsToSolidify.remove(); + } + + transactionsToSolidify.put(hash); + } + } catch(Exception e){ + log.error("Error placing transaction into solidification queue",e); + } + } + + @Override + public boolean addMilestoneToSolidificationQueue(Hash hash, int maxToProcess){ + try{ + TransactionViewModel tx = fromHash(tangle, hash); + if(tx.isSolid()){ + addToPropagationQueue(hash); + return true; + } + addToSolidificationQueue(hash); + return false; + }catch(Exception e){ + log.error("Error adding milestone to solidification queue", e); + return false; + } + } + + /** + *{@inheritDoc} + */ + @Override + public Set getBroadcastQueue(){ + return new LinkedHashSet<>(transactionsToBroadcast); + } + + /** + *{@inheritDoc} + */ + @Override + public void clearFromBroadcastQueue(Set transactionsBroadcasted){ + for (TransactionViewModel tvm : transactionsBroadcasted) { + transactionsToBroadcast.remove(tvm); + } + } + + + /** + * Iterate through the {@link #transactionsToSolidify} queue and call {@link #checkSolidity(Hash)} on each hash. + * Solid transactions are then processed into the {@link #transactionsToBroadcast} queue. + */ + private void processTransactionsToSolidify(){ + Hash hash; + while (!Thread.currentThread().isInterrupted()) { + if((hash = transactionsToSolidify.poll()) != null) { + try { + checkSolidity(hash); + } catch (Exception e) { + log.info(e.getMessage()); + } + } + propagateSolidTransactions(); + } + } + + /** + *{@inheritDoc} + */ + @Override + public boolean checkSolidity(Hash hash) throws Exception { + return checkSolidity(hash, 50000); + } + + /** + *{@inheritDoc} + */ + @Override + public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception { + if(fromHash(tangle, hash).isSolid()) { + return true; + } + LinkedHashSet analyzedHashes = new LinkedHashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); + if(maxProcessedTransactions != Integer.MAX_VALUE) { + maxProcessedTransactions += analyzedHashes.size(); + } + boolean solid = true; + final Queue nonAnalyzedTransactions = new LinkedList<>(Collections.singleton(hash)); + Hash hashPointer; + while ((hashPointer = nonAnalyzedTransactions.poll()) != null) { + if (!analyzedHashes.add(hashPointer)) { + continue; + } + + if (analyzedHashes.size() >= maxProcessedTransactions) { + return false; + } + + TransactionViewModel transaction = fromHash(tangle, hashPointer); + if (isUnsolidWithoutEntryPoint(transaction, hashPointer)) { + if (transaction.getType() == PREFILLED_SLOT) { + solid = false; + checkRequester(hashPointer); + } else { + nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); + nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); + } + } + } + + if (solid) { + updateTransactions(analyzedHashes); + } + analyzedHashes.clear(); + return solid; + } + + + /** + * Check if a transaction is present in the {@link #transactionRequester}, if not, it is added. + * @param hashPointer The hash of the transaction to request + */ + private void checkRequester(Hash hashPointer){ + if (!transactionRequester.isTransactionRequested(hashPointer)) { + transactionRequester.requestTransaction(hashPointer); + } + } + + /** + * Iterate through analyzed hashes and place them in the {@link #transactionsToBroadcast} queue + * @param hashes Analyzed hashes from the {@link #checkSolidity(Hash)} call + */ + private void updateTransactions(Set hashes) { + hashes.forEach(hash -> { + try { + TransactionViewModel tvm = fromHash(tangle, hash); + tvm.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); + + if(!tvm.isSolid()){ + tvm.updateSolid(true); + tvm.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); + } + addToBroadcastQueue(tvm); + addToPropagationQueue(tvm.getHash()); + } catch (Exception e) { + log.info(e.getMessage()); + } + }); + } + + /** + * Returns true if transaction is not solid and there are no solid entry points from the initial snapshot. + */ + private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Hash hashPointer) throws Exception{ + if(!transaction.isSolid()){ + if(!snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)){ + return true; + } + } + addToPropagationQueue(hashPointer); + return false; + } + + + private void addToBroadcastQueue(TransactionViewModel tvm) { + try { + if (transactionsToBroadcast.size() >= MAX_SIZE) { + transactionsToBroadcast.remove(); + } + + transactionsToBroadcast.put(tvm); + } catch(Exception e){ + log.info("Error placing transaction into broadcast queue: " + e.getMessage()); + } + } + + @VisibleForTesting + Set getSolidificationQueue(){ + return new LinkedHashSet<>(transactionsToSolidify); + } + + + @Override + public void updateStatus(TransactionViewModel transactionViewModel) throws Exception { + transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); + if(transactionViewModel.getApprovers(tangle).size() == 0) { + tipsViewModel.addTipHash(transactionViewModel.getHash()); + } + tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); + tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); + + if(quickSetSolid(transactionViewModel)) { + transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); + tipsViewModel.setSolid(transactionViewModel.getHash()); + addToPropagationQueue(transactionViewModel.getHash()); + } + } + + public void addToPropagationQueue(Hash hash) throws Exception{ + if(!solidTransactions.contains(hash)) { + if (solidTransactions.size() >= MAX_SIZE) { + solidTransactions.poll(); + } + solidTransactions.put(hash); + } + } + + @Override + public boolean quickSetSolid(final TransactionViewModel transactionViewModel) throws Exception { + if(!transactionViewModel.isSolid()) { + boolean solid = true; + if (!checkApproovee(transactionViewModel.getTrunkTransaction(tangle))) { + solid = false; + } + if (!checkApproovee(transactionViewModel.getBranchTransaction(tangle))) { + solid = false; + } + if(solid) { + transactionViewModel.updateSolid(true); + transactionViewModel.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); + addToPropagationQueue(transactionViewModel.getHash()); + addToBroadcastQueue(transactionViewModel); + return true; + } + } + return false; + } + + /** + * If the the {@code approvee} is missing, request it from a neighbor. + * @param approovee transaction we check. + * @return true if {@code approvee} is solid. + * @throws Exception if we encounter an error while requesting a transaction + */ + private boolean checkApproovee(TransactionViewModel approovee) throws Exception { + if(snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(approovee.getHash())) { + return true; + } + if(approovee.getType() == PREFILLED_SLOT) { + // don't solidify from the bottom until cuckoo filters can identify where we deleted -> otherwise we will + // continue requesting old transactions forever + //transactionRequester.requestTransaction(approovee.getHash(), false); + return false; + } + return approovee.isSolid(); + } + + @VisibleForTesting + void propagateSolidTransactions() { + Iterator cascadeIterator = solidTransactions.iterator(); + int cascadeCount = 0; + while(cascadeCount < MAX_SIZE && cascadeIterator.hasNext() && !Thread.currentThread().isInterrupted()) { + try { + cascadeCount += 1; + Hash hash = cascadeIterator.next(); + TransactionViewModel transaction = fromHash(tangle, hash); + Set approvers = transaction.getApprovers(tangle).getHashes(); + for(Hash h: approvers) { + TransactionViewModel tx = fromHash(tangle, h); + if (quietQuickSetSolid(tx)) { + tx.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); + tipsViewModel.setSolid(h); + } + } + cascadeIterator.remove(); + } catch (Exception e) { + log.error("Error while propagating solidity upwards", e); + } + } + } + + /** + * Perform a {@link #quickSetSolid} while capturing and logging errors + * @param transactionViewModel transaction we try to solidify. + * @return true if we managed to solidify, else false. + */ + private boolean quietQuickSetSolid(TransactionViewModel transactionViewModel) { + try { + return quickSetSolid(transactionViewModel); + } catch (Exception e) { + log.error(e.getMessage(), e); + return false; + } + } +} diff --git a/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java b/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java index 3695f6cdba..9d36056c14 100644 --- a/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java +++ b/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java @@ -12,8 +12,6 @@ import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.API; import com.iota.iri.service.ledger.LedgerService; -import com.iota.iri.service.milestone.LatestMilestoneTracker; -import com.iota.iri.service.milestone.LatestSolidMilestoneTracker; import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.milestone.SeenMilestonesRetriever; @@ -23,6 +21,7 @@ import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPruner; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.LocalSnapshotsPersistenceProvider; import com.iota.iri.storage.Tangle; import org.junit.Test; @@ -64,16 +63,6 @@ public void provideLedgerService() { assertNotNull("instance creation did not work", testInjector().getInstance(LedgerService.class)); } - @Test - public void provideLatestMilestoneTracker() { - assertNotNull("instance creation did not work", testInjector().getInstance(LatestMilestoneTracker.class)); - } - - @Test - public void provideLatestSolidMilestoneTracker() { - assertNotNull("instance creation did not work", testInjector().getInstance(LatestSolidMilestoneTracker.class)); - } - @Test public void provideSeenMilestonesRetriever() { assertNotNull("instance creation did not work", testInjector().getInstance(SeenMilestonesRetriever.class)); diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java deleted file mode 100644 index ad35624978..0000000000 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ /dev/null @@ -1,224 +0,0 @@ -package com.iota.iri; - -import com.iota.iri.conf.MainnetConfig; -import com.iota.iri.conf.ProtocolConfig; - -import com.iota.iri.controllers.TipsViewModel; -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.crypto.SpongeFactory; -import com.iota.iri.model.TransactionHash; -import com.iota.iri.network.TransactionRequester; -import com.iota.iri.service.snapshot.SnapshotProvider; - -import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; -import com.iota.iri.storage.Tangle; -import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; -import com.iota.iri.utils.Converter; - -import org.junit.*; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; - -import static com.iota.iri.TransactionTestUtils.getTransactionHash; -import static com.iota.iri.TransactionTestUtils.getTransactionTrits; -import static com.iota.iri.TransactionTestUtils.getTransactionTritsWithTrunkAndBranch; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class TransactionValidatorTest { - - private static final int MAINNET_MWM = 14; - private static final TemporaryFolder dbFolder = new TemporaryFolder(); - private static final TemporaryFolder logFolder = new TemporaryFolder(); - private static Tangle tangle; - private static TransactionValidator txValidator; - - @Rule - public MockitoRule mockitoRule = MockitoJUnit.rule(); - - @Mock - private static SnapshotProvider snapshotProvider; - - @BeforeClass - public static void setUp() throws Exception { - dbFolder.create(); - logFolder.create(); - tangle = new Tangle(); - tangle.addPersistenceProvider( - new RocksDBPersistenceProvider( - dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); - tangle.init(); - } - - @AfterClass - public static void tearDown() throws Exception { - tangle.shutdown(); - dbFolder.delete(); - logFolder.delete(); - } - - @Before - public void setUpEach() { - when(snapshotProvider.getInitialSnapshot()).thenReturn(SnapshotMockUtils.createSnapshot()); - TipsViewModel tipsViewModel = new TipsViewModel(); - TransactionRequester txRequester = new TransactionRequester(tangle, snapshotProvider); - txValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, txRequester, new MainnetConfig()); - txValidator.setMwm(false, MAINNET_MWM); - } - - @Test - public void testMinMwm() { - ProtocolConfig protocolConfig = mock(ProtocolConfig.class); - when(protocolConfig.getMwm()).thenReturn(5); - TransactionValidator transactionValidator = new TransactionValidator(null, null, null, null, protocolConfig); - assertEquals("Expected testnet minimum minWeightMagnitude", 13, transactionValidator.getMinWeightMagnitude()); - } - - @Test - public void validateTrits() { - byte[] trits = getTransactionTrits(); - Converter.copyTrits(0, trits, 0, trits.length); - txValidator.validateTrits(trits, MAINNET_MWM); - } - - @Test(expected = RuntimeException.class) - public void validateTritsWithInvalidMetadata() { - byte[] trits = getTransactionTrits(); - txValidator.validateTrits(trits, MAINNET_MWM); - } - - @Test - public void validateBytesWithNewCurl() { - byte[] trits = getTransactionTrits(); - Converter.copyTrits(0, trits, 0, trits.length); - byte[] bytes = Converter.allocateBytesForTrits(trits.length); - Converter.bytes(trits, 0, bytes, 0, trits.length); - txValidator.validateBytes(bytes, txValidator.getMinWeightMagnitude(), SpongeFactory.create(SpongeFactory.Mode.CURLP81)); - } - - @Test - public void verifyTxIsSolid() throws Exception { - TransactionViewModel tx = getTxWithBranchAndTrunk(); - assertTrue(txValidator.checkSolidity(tx.getHash())); - assertTrue(txValidator.checkSolidity(tx.getHash())); - } - - @Test - public void verifyTxIsNotSolid() throws Exception { - TransactionViewModel tx = getTxWithoutBranchAndTrunk(); - assertFalse(txValidator.checkSolidity(tx.getHash())); - assertFalse(txValidator.checkSolidity(tx.getHash())); - } - - @Test - public void addSolidTransactionWithoutErrors() { - byte[] trits = getTransactionTrits(); - Converter.copyTrits(0, trits, 0, trits.length); - txValidator.addSolidTransaction(TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); - } - - private TransactionViewModel getTxWithBranchAndTrunk() throws Exception { - TransactionViewModel tx, trunkTx, branchTx; - String trytes = "999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999CFDEZBLZQYA9999999999999999999999999999999999999999999ZZWQHWD99C99999999C99999999CKWWDBWSCLMQULCTAAJGXDEMFJXPMGMAQIHDGHRBGEMUYNNCOK9YPHKEEFLFCZUSPMCJHAKLCIBQSGWAS999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999"; - - byte[] trits = Converter.allocateTritsForTrytes(trytes.length()); - Converter.trits(trytes, trits, 0); - trunkTx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); - branchTx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); - - byte[] childTx = getTransactionTrits(); - System.arraycopy(trunkTx.getHash().trits(), 0, childTx, TransactionViewModel.TRUNK_TRANSACTION_TRINARY_OFFSET, TransactionViewModel.TRUNK_TRANSACTION_TRINARY_SIZE); - System.arraycopy(branchTx.getHash().trits(), 0, childTx, TransactionViewModel.BRANCH_TRANSACTION_TRINARY_OFFSET, TransactionViewModel.BRANCH_TRANSACTION_TRINARY_SIZE); - tx = new TransactionViewModel(childTx, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, childTx)); - - trunkTx.store(tangle, snapshotProvider.getInitialSnapshot()); - branchTx.store(tangle, snapshotProvider.getInitialSnapshot()); - tx.store(tangle, snapshotProvider.getInitialSnapshot()); - - return tx; - } - - @Test - public void testTransactionPropagation() throws Exception { - TransactionViewModel leftChildLeaf = TransactionTestUtils.createTransactionWithTrytes("CHILDTX"); - leftChildLeaf.updateSolid(true); - leftChildLeaf.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel rightChildLeaf = TransactionTestUtils.createTransactionWithTrytes("CHILDTWOTX"); - rightChildLeaf.updateSolid(true); - rightChildLeaf.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel parent = TransactionTestUtils.createTransactionWithTrunkAndBranch("PARENT", - leftChildLeaf.getHash(), rightChildLeaf.getHash()); - parent.updateSolid(false); - parent.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel parentSibling = TransactionTestUtils.createTransactionWithTrytes("PARENTLEAF"); - parentSibling.updateSolid(true); - parentSibling.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel grandParent = TransactionTestUtils.createTransactionWithTrunkAndBranch("GRANDPARENT", parent.getHash(), - parentSibling.getHash()); - grandParent.updateSolid(false); - grandParent.store(tangle, snapshotProvider.getInitialSnapshot()); - - txValidator.addSolidTransaction(leftChildLeaf.getHash()); - while (!txValidator.isNewSolidTxSetsEmpty()) { - txValidator.propagateSolidTransactions(); - } - - parent = TransactionViewModel.fromHash(tangle, parent.getHash()); - assertTrue("Parent tx was expected to be solid", parent.isSolid()); - grandParent = TransactionViewModel.fromHash(tangle, grandParent.getHash()); - assertTrue("Grandparent was expected to be solid", grandParent.isSolid()); - } - - @Test - public void testTransactionPropagationFailure() throws Exception { - TransactionViewModel leftChildLeaf = new TransactionViewModel(getTransactionTrits(), getTransactionHash()); - leftChildLeaf.updateSolid(true); - leftChildLeaf.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel rightChildLeaf = new TransactionViewModel(getTransactionTrits(), getTransactionHash()); - rightChildLeaf.updateSolid(true); - rightChildLeaf.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel parent = new TransactionViewModel(getTransactionTritsWithTrunkAndBranch(leftChildLeaf.getHash(), - rightChildLeaf.getHash()), getTransactionHash()); - parent.updateSolid(false); - parent.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel parentSibling = new TransactionViewModel(getTransactionTrits(), getTransactionHash()); - parentSibling.updateSolid(false); - parentSibling.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel grandParent = new TransactionViewModel(getTransactionTritsWithTrunkAndBranch(parent.getHash(), - parentSibling.getHash()), getTransactionHash()); - grandParent.updateSolid(false); - grandParent.store(tangle, snapshotProvider.getInitialSnapshot()); - - txValidator.addSolidTransaction(leftChildLeaf.getHash()); - while (!txValidator.isNewSolidTxSetsEmpty()) { - txValidator.propagateSolidTransactions(); - } - - parent = TransactionViewModel.fromHash(tangle, parent.getHash()); - assertTrue("Parent tx was expected to be solid", parent.isSolid()); - grandParent = TransactionViewModel.fromHash(tangle, grandParent.getHash()); - assertFalse("GrandParent tx was expected to be not solid", grandParent.isSolid()); - } - - private TransactionViewModel getTxWithoutBranchAndTrunk() throws Exception { - byte[] trits = getTransactionTrits(); - TransactionViewModel tx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); - - tx.store(tangle, snapshotProvider.getInitialSnapshot()); - - return tx; - } -} diff --git a/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java b/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java index c02749a893..a3333d0c04 100644 --- a/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java +++ b/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java @@ -3,11 +3,13 @@ import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.BaseIotaConfig; import com.iota.iri.conf.IotaConfig; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import org.junit.Test; @@ -37,6 +39,11 @@ public void provideTransactionProcessingPipeline() { assertNotNull("instance creation did not work", testInjector().getInstance(TransactionProcessingPipeline.class)); } + @Test + public void provideTransactionSolidifier(){ + assertNotNull("instance creation did not work", testInjector().getInstance(TransactionSolidifier.class)); + } + private Injector testInjector() { IotaConfig config = mock(IotaConfig.class); when(config.getCoordinator()).thenReturn(BaseIotaConfig.Defaults.COORDINATOR); @@ -47,9 +54,11 @@ private class MockedDependencies extends AbstractModule { @Override protected void configure() { - bind(LatestMilestoneTracker.class).toInstance(mock(LatestMilestoneTracker.class)); + bind(MilestoneSolidifier.class).toInstance(mock(MilestoneSolidifier.class)); bind(SnapshotProvider.class).toInstance(mock(SnapshotProvider.class)); bind(TransactionValidator.class).toInstance(mock(TransactionValidator.class)); + bind(TransactionSolidifier.class).toInstance(mock(TransactionSolidifier.class)); + bind(MilestoneService.class).toInstance(mock(MilestoneService.class)); } } diff --git a/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java b/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java index 772295c4e5..b6a40412c2 100644 --- a/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java @@ -1,11 +1,13 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.model.Hash; +import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.Neighbor; import com.iota.iri.network.neighbor.impl.NeighborMetricsImpl; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.storage.Tangle; import org.junit.Rule; @@ -26,7 +28,7 @@ public class ReceivedStageTest { private Tangle tangle; @Mock - private TransactionValidator transactionValidator; + private TransactionSolidifier txSolidifier; @Mock private SnapshotProvider snapshotProvider; @@ -43,13 +45,19 @@ public class ReceivedStageTest { @Mock private NeighborMetricsImpl neighborMetrics; + @Mock + private MilestoneService milestoneService; + + @Mock + private Hash cooAddress; + @Test public void newlyStoredTransactionUpdatesAlsoArrivalTimeAndSender() throws Exception { Mockito.when(tvm.store(tangle, snapshotProvider.getInitialSnapshot())).thenReturn(true); Mockito.when(neighbor.getMetrics()).thenReturn(neighborMetrics); Mockito.when(transactionRequester.removeRecentlyRequestedTransaction(Mockito.any())).thenReturn(true); - ReceivedStage stage = new ReceivedStage(tangle, transactionValidator, snapshotProvider, transactionRequester); + ReceivedStage stage = new ReceivedStage(tangle, txSolidifier, snapshotProvider, transactionRequester, milestoneService, cooAddress); ReceivedPayload receivedPayload = new ReceivedPayload(neighbor, tvm); ProcessingContext ctx = new ProcessingContext(null, receivedPayload); stage.process(ctx); @@ -58,11 +66,11 @@ public void newlyStoredTransactionUpdatesAlsoArrivalTimeAndSender() throws Excep Mockito.verify(tvm).update(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(transactionRequester).removeRecentlyRequestedTransaction(Mockito.any()); Mockito.verify(transactionRequester).requestTrunkAndBranch(Mockito.any()); - assertEquals("should submit to broadcast stage next", TransactionProcessingPipeline.Stage.BROADCAST, + assertEquals("should submit to broadcast stage next", TransactionProcessingPipeline.Stage.SOLIDIFY, ctx.getNextStage()); - BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); - assertEquals("neighbor is still the same", neighbor, broadcastPayload.getOriginNeighbor()); - assertEquals("tvm is still the same", tvm, broadcastPayload.getTransactionViewModel()); + SolidifyPayload solidifyPayload = (SolidifyPayload) ctx.getPayload(); + assertEquals("neighbor is still the same", neighbor, solidifyPayload.getOriginNeighbor()); + assertEquals("tvm is still the same", tvm, solidifyPayload.getTransaction()); } @Test @@ -70,7 +78,7 @@ public void alreadyStoredTransactionDoesNoUpdates() throws Exception { Mockito.when(tvm.store(tangle, snapshotProvider.getInitialSnapshot())).thenReturn(false); Mockito.when(neighbor.getMetrics()).thenReturn(neighborMetrics); - ReceivedStage stage = new ReceivedStage(tangle, transactionValidator, snapshotProvider, transactionRequester); + ReceivedStage stage = new ReceivedStage(tangle, txSolidifier, snapshotProvider, transactionRequester, milestoneService, cooAddress); ReceivedPayload receivedPayload = new ReceivedPayload(neighbor, tvm); ProcessingContext ctx = new ProcessingContext(null, receivedPayload); stage.process(ctx); @@ -79,11 +87,11 @@ public void alreadyStoredTransactionDoesNoUpdates() throws Exception { Mockito.verify(tvm, Mockito.never()).update(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(transactionRequester).removeRecentlyRequestedTransaction(Mockito.any()); Mockito.verify(transactionRequester, Mockito.never()).requestTrunkAndBranch(Mockito.any()); - assertEquals("should submit to broadcast stage next", TransactionProcessingPipeline.Stage.BROADCAST, + assertEquals("should submit to broadcast stage next", TransactionProcessingPipeline.Stage.SOLIDIFY, ctx.getNextStage()); - BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); - assertEquals("neighbor should still be the same", neighbor, broadcastPayload.getOriginNeighbor()); - assertEquals("tvm should still be the same", tvm, broadcastPayload.getTransactionViewModel()); + SolidifyPayload solidifyPayload = (SolidifyPayload) ctx.getPayload(); + assertEquals("neighbor should still be the same", neighbor, solidifyPayload.getOriginNeighbor()); + assertEquals("tvm should still be the same", tvm, solidifyPayload.getTransaction()); } } \ No newline at end of file diff --git a/src/test/java/com/iota/iri/network/pipeline/ReplyStageTest.java b/src/test/java/com/iota/iri/network/pipeline/ReplyStageTest.java index fc1ba0f2bc..96eff3009b 100644 --- a/src/test/java/com/iota/iri/network/pipeline/ReplyStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/ReplyStageTest.java @@ -10,7 +10,7 @@ import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.impl.NeighborImpl; import com.iota.iri.network.neighbor.impl.NeighborMetricsImpl; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; @@ -44,7 +44,7 @@ public class ReplyStageTest { private TipsViewModel tipsViewModel; @Mock - private LatestMilestoneTracker latestMilestoneTracker; + private MilestoneSolidifier milestoneSolidifier; @Mock private SnapshotProvider snapshotProvider; @@ -74,13 +74,13 @@ public void usingTheNullHashARandomTipIsGettingReplied() { Mockito.when(neighbor.getMetrics()).thenReturn(neighborMetrics); Mockito.when(snapshotProvider.getLatestSnapshot()).thenReturn(snapshot); Mockito.when(snapshot.getIndex()).thenReturn(8); - Mockito.when(latestMilestoneTracker.getLatestMilestoneIndex()).thenReturn(10); + Mockito.when(milestoneSolidifier.getLatestMilestoneIndex()).thenReturn(10); Mockito.when(transactionRequester.numberOfTransactionsToRequest()).thenReturn(1); Mockito.when(tipsViewModel.getRandomSolidTipHash()).thenReturn(SampleTransaction.CURL_HASH_OF_SAMPLE_TX); TangleMockUtils.mockTransaction(tangle, SampleTransaction.CURL_HASH_OF_SAMPLE_TX, SampleTransaction.SAMPLE_TRANSACTION); - ReplyStage stage = new ReplyStage(neighborRouter, nodeConfig, tangle, tipsViewModel, latestMilestoneTracker, + ReplyStage stage = new ReplyStage(neighborRouter, nodeConfig, tangle, tipsViewModel, milestoneSolidifier, snapshotProvider, recentlySeenBytesCache, random); ReplyPayload replyPayload = new ReplyPayload(neighbor, Hash.NULL_HASH); ProcessingContext ctx = new ProcessingContext(replyPayload); @@ -100,7 +100,7 @@ public void usingASuppliedRequestedHashTheTransactionIsReplied() { TangleMockUtils.mockTransaction(tangle, SampleTransaction.CURL_HASH_OF_SAMPLE_TX, SampleTransaction.SAMPLE_TRANSACTION); - ReplyStage stage = new ReplyStage(neighborRouter, nodeConfig, tangle, tipsViewModel, latestMilestoneTracker, + ReplyStage stage = new ReplyStage(neighborRouter, nodeConfig, tangle, tipsViewModel, milestoneSolidifier, snapshotProvider, recentlySeenBytesCache, random); ReplyPayload replyPayload = new ReplyPayload(neighbor, SampleTransaction.CURL_HASH_OF_SAMPLE_TX); ProcessingContext ctx = new ProcessingContext(replyPayload); diff --git a/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java b/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java new file mode 100644 index 0000000000..958c5f3259 --- /dev/null +++ b/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java @@ -0,0 +1,115 @@ +package com.iota.iri.network.pipeline; + +import com.iota.iri.controllers.TipsViewModel; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.model.persistables.Transaction; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.storage.Tangle; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import static org.junit.Assert.assertEquals; + +public class SolidifyStageTest { + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private Tangle tangle; + + @Mock + private TipsViewModel tipsViewModel; + + @Mock + private TransactionViewModel tvm; + + @Mock + private Hash originalHash; + + @Mock + private Hash tipHash; + + @Mock + private TransactionSolidifier transactionSolidifier; + + @Test + public void solidTransactionIsBroadcast() throws Exception{ + Mockito.when(tvm.isSolid()).thenReturn(true); + Mockito.when(tvm.getHash()).thenReturn(originalHash); + + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, tipsViewModel, tangle); + SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); + ProcessingContext ctx = new ProcessingContext(solidifyPayload); + + solidifyStage.process(ctx); + Thread.sleep(100); + + assertEquals("Expected next stage to be broadcast", ctx.getNextStage(), + TransactionProcessingPipeline.Stage.BROADCAST); + BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); + assertEquals("Expected payload hash to equal the original transaction hash", + broadcastPayload.getTransactionViewModel().getHash(), originalHash); + } + + @Test + public void quickSetSolidTransactionIsBroadcast() throws Exception{ + Mockito.when(transactionSolidifier.quickSetSolid(tvm)).thenReturn(true); + Mockito.when(tvm.getHash()).thenReturn(originalHash); + + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, tipsViewModel, tangle); + SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); + ProcessingContext ctx = new ProcessingContext(solidifyPayload); + + solidifyStage.process(ctx); + Thread.sleep(100); + + assertEquals("Expected next stage to be broadcast", ctx.getNextStage(), + TransactionProcessingPipeline.Stage.BROADCAST); + BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); + assertEquals("Expected payload hash to equal the original transaction hash", + broadcastPayload.getTransactionViewModel().getHash(), originalHash); + } + + @Test + public void unsolidTransactionBroadcastsRandomSolidTip() throws Exception{ + Mockito.when(tvm.isSolid()).thenReturn(false); + Mockito.when(transactionSolidifier.quickSetSolid(tvm)).thenReturn(false); + TransactionViewModel tip = new TransactionViewModel(new Transaction(), tipHash); + + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, tipsViewModel, tangle); + SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); + ProcessingContext ctx = new ProcessingContext(solidifyPayload); + + solidifyStage.injectTip(tip); + solidifyStage.process(ctx); + Thread.sleep(100); + + assertEquals("Expected next stage to be broadcast", ctx.getNextStage(), + TransactionProcessingPipeline.Stage.BROADCAST); + BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); + assertEquals("Expected payload hash to equal random tip hash", + broadcastPayload.getTransactionViewModel().getHash(), tipHash); + } + + @Test + public void unsolidWithNoRandomTipsAborts() throws Exception{ + Mockito.when(tvm.isSolid()).thenReturn(false); + Mockito.when(transactionSolidifier.quickSetSolid(tvm)).thenReturn(false); + Mockito.when(tipsViewModel.getRandomSolidTipHash()).thenReturn(null); + + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, tipsViewModel, tangle); + SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); + ProcessingContext ctx = new ProcessingContext(solidifyPayload); + + solidifyStage.process(ctx); + Thread.sleep(100); + + assertEquals("Expected next stage to be broadcast", ctx.getNextStage(), + TransactionProcessingPipeline.Stage.FINISH); + } +} diff --git a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java index b9b8c4ab8f..2e6436353e 100644 --- a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java @@ -1,13 +1,15 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.NodeConfig; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.network.NeighborRouter; import com.iota.iri.network.SampleTransaction; import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.Neighbor; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; @@ -18,6 +20,7 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; + public class TransactionProcessingPipelineTest { @Rule @@ -41,9 +44,6 @@ public class TransactionProcessingPipelineTest { @Mock private TipsViewModel tipsViewModel; - @Mock - private LatestMilestoneTracker latestMilestoneTracker; - @Mock private TransactionRequester transactionRequester; @@ -68,12 +68,27 @@ public class TransactionProcessingPipelineTest { @Mock private HashingStage hashingStage; + @Mock + private MilestoneStage milestoneStage; + @Mock private ProcessingContext hashingCtx; @Mock private HashingPayload hashingPayload; + @Mock + private BroadcastPayload broadcastPayload; + + @Mock + private SolidifyStage solidifyStage; + + @Mock + private SolidifyPayload solidifyPayload; + + @Mock + private MilestonePayload milestonePayload; + @Mock private ProcessingContext validationCtx; @@ -89,9 +104,25 @@ public class TransactionProcessingPipelineTest { @Mock private ProcessingContext broadcastCtx; + @Mock + private ProcessingContext solidifyCtx; + @Mock private ProcessingContext abortCtx; + @Mock + private ProcessingContext milestoneCtx; + + @Mock + private MilestoneService milestoneService; + + @Mock + private MilestoneSolidifier milestoneSolidifier; + + + @Mock + private TransactionSolidifier transactionSolidifier; + private void mockHashingStage(TransactionProcessingPipeline pipeline) { Mockito.when(hashingPayload.getTxTrits()).thenReturn(null); Mockito.doAnswer(invocation -> { @@ -107,14 +138,16 @@ private void injectMockedStagesIntoPipeline(TransactionProcessingPipeline pipeli pipeline.setHashingStage(hashingStage); pipeline.setReplyStage(replyStage); pipeline.setValidationStage(validationStage); + pipeline.setSolidifyStage(solidifyStage); + pipeline.setMilestoneStage(milestoneStage); } @Test public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, - transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester); + transactionValidator, tangle, snapshotProvider, tipsViewModel, transactionRequester, + transactionSolidifier, milestoneService, milestoneSolidifier); // inject mocks injectMockedStagesIntoPipeline(pipeline); @@ -136,7 +169,13 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws // mock received Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); - Mockito.when(receivedStage.process(receivedCtx)).thenReturn(broadcastCtx); + Mockito.when(broadcastCtx.getPayload()).thenReturn(broadcastPayload); + Mockito.when(receivedStage.process(receivedCtx)).thenReturn(solidifyCtx); + + // mock solidify + Mockito.when(solidifyCtx.getPayload()).thenReturn(solidifyPayload); + Mockito.when(solidifyCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.SOLIDIFY); + Mockito.when(solidifyStage.process(solidifyCtx)).thenReturn(broadcastCtx); pipeline.start(); @@ -152,14 +191,72 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws Mockito.verify(validationStage).process(Mockito.any()); Mockito.verify(receivedStage).process(Mockito.any()); Mockito.verify(replyStage).process(Mockito.any()); + Mockito.verify(solidifyStage).process(Mockito.any()); Mockito.verify(broadcastStage).process(Mockito.any()); } + @Test + public void processingAValidMilestone() throws InterruptedException { + TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, + transactionValidator, tangle, snapshotProvider, tipsViewModel, transactionRequester, + transactionSolidifier, milestoneService, milestoneSolidifier); + + injectMockedStagesIntoPipeline(pipeline); + + // mock after pre process context/stage + Mockito.when(preProcessStage.process(Mockito.any())).thenReturn(hashingCtx); + Mockito.when(hashingCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.HASHING); + Mockito.when(hashingCtx.getPayload()).thenReturn(hashingPayload); + + // mock hashing context/stage + mockHashingStage(pipeline); + + // mock validation context/stage + MultiStagePayload divergePayload = new MultiStagePayload(replyCtx, receivedCtx); + Mockito.when(validationStage.process(validationCtx)).thenReturn(divergeToReplyAndReceivedCtx); + Mockito.when(divergeToReplyAndReceivedCtx.getNextStage()) + .thenReturn(TransactionProcessingPipeline.Stage.MULTIPLE); + Mockito.when(divergeToReplyAndReceivedCtx.getPayload()).thenReturn(divergePayload); + + // mock received + Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); + Mockito.when(broadcastCtx.getPayload()).thenReturn(broadcastPayload); + Mockito.when(receivedStage.process(receivedCtx)).thenReturn(milestoneCtx); + + // mock milestone + Mockito.when(milestoneCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.MILESTONE); + Mockito.when(milestoneCtx.getPayload()).thenReturn(milestonePayload); + Mockito.when(milestoneStage.process(milestoneCtx)).thenReturn(solidifyCtx); + + // mock solidify + Mockito.when(solidifyCtx.getPayload()).thenReturn(solidifyPayload); + Mockito.when(solidifyCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.SOLIDIFY); + Mockito.when(solidifyStage.process(solidifyCtx)).thenReturn(broadcastCtx); + + pipeline.start(); + + // send in actual payload to kick off the 'processing' + pipeline.process(neighbor, SampleTransaction.createSampleTxBuffer()); + + // give it some time to 'process' + Thread.sleep(100); + + // should have called + Mockito.verify(preProcessStage).process(Mockito.any()); + Mockito.verify(hashingStage).process(Mockito.any()); + Mockito.verify(validationStage).process(Mockito.any()); + Mockito.verify(receivedStage).process(Mockito.any()); + Mockito.verify(replyStage).process(Mockito.any()); + Mockito.verify(solidifyStage).process(Mockito.any()); + Mockito.verify(broadcastStage).process(Mockito.any()); + Mockito.verify(milestoneStage).process(Mockito.any()); + } + @Test public void processingAKnownTransactionOnlyFlowsToTheReplyStage() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, - transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester); + transactionValidator, tangle, snapshotProvider, tipsViewModel, transactionRequester, + transactionSolidifier, milestoneService, milestoneSolidifier); // inject mocks pipeline.setPreProcessStage(preProcessStage); @@ -181,6 +278,7 @@ public void processingAKnownTransactionOnlyFlowsToTheReplyStage() throws Interru Mockito.verify(hashingStage, Mockito.never()).process(Mockito.any()); Mockito.verify(validationStage, Mockito.never()).process(Mockito.any()); Mockito.verify(receivedStage, Mockito.never()).process(Mockito.any()); + Mockito.verify(solidifyStage, Mockito.never()).process(Mockito.any()); Mockito.verify(broadcastStage, Mockito.never()).process(Mockito.any()); // should have called @@ -192,8 +290,8 @@ public void processingAKnownTransactionOnlyFlowsToTheReplyStage() throws Interru public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroughTheCorrectStages() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, - transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester); + transactionValidator, tangle, snapshotProvider, tipsViewModel, transactionRequester, + transactionSolidifier, milestoneService, milestoneSolidifier); // inject mocks injectMockedStagesIntoPipeline(pipeline); @@ -202,7 +300,6 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug Mockito.when(hashingCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.HASHING); Mockito.when(hashingCtx.getPayload()).thenReturn(hashingPayload); - // mock hashing context/stage // mock hashing context/stage mockHashingStage(pipeline); @@ -212,7 +309,12 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug // mock received Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); - Mockito.when(receivedStage.process(receivedCtx)).thenReturn(broadcastCtx); + Mockito.when(receivedStage.process(receivedCtx)).thenReturn(solidifyCtx); + + // mock solidify + Mockito.when(solidifyCtx.getPayload()).thenReturn(solidifyPayload); + Mockito.when(solidifyCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.SOLIDIFY); + Mockito.when(solidifyStage.process(solidifyCtx)).thenReturn(broadcastCtx); pipeline.start(); @@ -231,14 +333,15 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug Mockito.verify(hashingStage).process(Mockito.any()); Mockito.verify(validationStage).process(Mockito.any()); Mockito.verify(receivedStage).process(Mockito.any()); + Mockito.verify(solidifyStage).process(Mockito.any()); Mockito.verify(broadcastStage).process(Mockito.any()); } @Test public void anInvalidNewTransactionStopsBeingProcessedAfterTheValidationStage() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, - transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester); + transactionValidator, tangle, snapshotProvider, tipsViewModel, transactionRequester, + transactionSolidifier, milestoneService, milestoneSolidifier); // inject mocks injectMockedStagesIntoPipeline(pipeline); @@ -269,6 +372,7 @@ public void anInvalidNewTransactionStopsBeingProcessedAfterTheValidationStage() Mockito.verify(preProcessStage, Mockito.never()).process(Mockito.any()); Mockito.verify(broadcastStage, Mockito.never()).process(Mockito.any()); Mockito.verify(receivedStage, Mockito.never()).process(Mockito.any()); + Mockito.verify(solidifyStage, Mockito.never()).process(Mockito.any()); // should have called Mockito.verify(hashingStage).process(Mockito.any()); diff --git a/src/test/java/com/iota/iri/network/pipeline/ValidationStageTest.java b/src/test/java/com/iota/iri/network/pipeline/ValidationStageTest.java index 5bd0eecdff..c02d57d3b4 100644 --- a/src/test/java/com/iota/iri/network/pipeline/ValidationStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/ValidationStageTest.java @@ -1,6 +1,6 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.network.FIFOCache; diff --git a/src/test/java/com/iota/iri/service/APITest.java b/src/test/java/com/iota/iri/service/APITest.java index f5639156fb..7169f339ad 100644 --- a/src/test/java/com/iota/iri/service/APITest.java +++ b/src/test/java/com/iota/iri/service/APITest.java @@ -1,6 +1,7 @@ package com.iota.iri.service; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.IotaConfig; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.service.snapshot.SnapshotProvider; @@ -28,6 +29,9 @@ public class APITest { @Mock(answer = Answers.RETURNS_SMART_NULLS) private TransactionValidator transactionValidator; + @Mock + private TransactionSolidifier transactionSolidifier; + @Mock private SnapshotProvider snapshotProvider; @@ -43,7 +47,7 @@ public void whenStoreTransactionsStatementThenSetArrivalTimeToCurrentMillis() th API api = new API(config, null, null, null, null, null, snapshotProvider, null, null, null, null, - transactionValidator, null, null); + transactionValidator, null, null, transactionSolidifier); api.storeTransactionsStatement(Collections.singletonList("FOO")); diff --git a/src/test/java/com/iota/iri/service/ApiCallTest.java b/src/test/java/com/iota/iri/service/ApiCallTest.java index a6f4f0de39..d2ca7dc9b0 100644 --- a/src/test/java/com/iota/iri/service/ApiCallTest.java +++ b/src/test/java/com/iota/iri/service/ApiCallTest.java @@ -14,7 +14,7 @@ public class ApiCallTest { @Before public void setUp() { IotaConfig configuration = Mockito.mock(IotaConfig.class); - api = new API(configuration, null, null, null, null, null, null, null, null, null, null, null, null, null); + api = new API(configuration, null, null, null, null, null, null, null, null, null, null, null, null, null, null); } @Test diff --git a/src/test/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImplTest.java b/src/test/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImplTest.java index ad29ad6fab..148085a283 100644 --- a/src/test/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImplTest.java +++ b/src/test/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImplTest.java @@ -3,7 +3,6 @@ import com.iota.iri.TangleMockUtils; import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.SnapshotConfig; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.SnapshotService; import com.iota.iri.service.snapshot.conditions.SnapshotDepthCondition; @@ -11,6 +10,7 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.utils.thread.ThreadUtils; +import com.iota.iri.service.milestone.MilestoneSolidifier; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -57,7 +57,7 @@ public class LocalSnapshotManagerImplTest { Tangle tangle; @Mock(answer = Answers.RETURNS_DEEP_STUBS) - LatestMilestoneTracker milestoneTracker; + MilestoneSolidifier milestoneSolidifier; private LocalSnapshotManagerImpl lsManager; @@ -84,17 +84,16 @@ public synchronized void takeLocalSnapshot() throws Exception { TangleMockUtils.mockMilestone(tangle, TransactionTestUtils.getTransactionHash(), 100-SNAPSHOT_DEPTH-1); // Always return true - when(milestoneTracker.isInitialScanComplete()).thenReturn(true); + when(milestoneSolidifier.isInitialScanComplete()).thenReturn(true); // When we call it, we are in sync - when(milestoneTracker.getLatestMilestoneIndex()).thenReturn(200); - + when(milestoneSolidifier.getLatestMilestoneIndex()).thenReturn(200); // We are more then the depth ahead when(snapshotProvider.getLatestSnapshot().getIndex()).thenReturn(100); when(snapshotProvider.getInitialSnapshot().getIndex()).thenReturn(100 - SNAPSHOT_DEPTH - DELAY_SYNC - 1); // Run in separate thread to allow us to time-out - Thread t = new Thread(() -> lsManager.monitorThread(milestoneTracker)); + Thread t = new Thread(() -> lsManager.monitorThread(milestoneSolidifier)); t.start(); // We should finish directly, margin for slower computers @@ -113,39 +112,39 @@ public synchronized void takeLocalSnapshot() throws Exception { @Test public void isInSyncTestScanIncomplete() { - when(milestoneTracker.isInitialScanComplete()).thenReturn(false); + when(milestoneSolidifier.isInitialScanComplete()).thenReturn(false); - assertFalse("We should be out of sync when he havent finished initial scans", lsManager.isInSync(milestoneTracker)); + assertFalse("We should be out of sync when he havent finished initial scans", lsManager.isInSync(milestoneSolidifier)); } @Test public void isInSyncTestScanComplete() { // Always return true - when(milestoneTracker.isInitialScanComplete()).thenReturn(true); + when(milestoneSolidifier.isInitialScanComplete()).thenReturn(true); // We don't really support -1 indexes, but if this breaks, it is a good indication to be careful going further - when(milestoneTracker.getLatestMilestoneIndex()).thenReturn(-1, 5, 10, 998 + BUFFER - 1, 2000); + when(milestoneSolidifier.getLatestMilestoneIndex()).thenReturn(-1, 5, 10, 998 + BUFFER - 1, 2000); // snapshotProvider & milestoneTracker // -5 & -1 -> not in sync - assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneTracker)); + assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneSolidifier)); // -1 and 5 -> not in sync - assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneTracker)); + assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneSolidifier)); // 10 and 10 -> in sync - assertTrue("Equal index should be in sync", lsManager.isInSync(milestoneTracker)); + assertTrue("Equal index should be in sync", lsManager.isInSync(milestoneSolidifier)); // 998 and 1002 -> in sync since sync gap = 5 - assertTrue("Index difference less than the buffer still should be in sync", lsManager.isInSync(milestoneTracker)); + assertTrue("Index difference less than the buffer still should be in sync", lsManager.isInSync(milestoneSolidifier)); // 999 and 2000 -> out of sync again, bigger gap than 5 - assertFalse("Index difference more than the buffer should be out of sync again ", lsManager.isInSync(milestoneTracker)); + assertFalse("Index difference more than the buffer should be out of sync again ", lsManager.isInSync(milestoneSolidifier)); // 1999 and 2000 -> out of sync still - assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneTracker)); + assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneSolidifier)); // 2000 and 2000 -> in sync again - assertTrue("Equal index should be in sync", lsManager.isInSync(milestoneTracker)); + assertTrue("Equal index should be in sync", lsManager.isInSync(milestoneSolidifier)); } } diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImplTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImplTest.java index 8459ccfb7b..b8709a2d1c 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImplTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImplTest.java @@ -4,7 +4,7 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.IntegerIndex; import com.iota.iri.model.TransactionHash; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; import com.iota.iri.service.tipselection.EntryPointSelector; @@ -24,7 +24,7 @@ public class EntryPointSelectorImplTest { public MockitoRule mockitoRule = MockitoJUnit.rule(); @Mock - private LatestMilestoneTracker latestMilestoneTracker; + private MilestoneSolidifier milestoneSolidifier; @Mock private Tangle tangle; @@ -44,7 +44,7 @@ public void testEntryPointBWithTangleData() throws Exception { mockTangleBehavior(milestoneHash); mockMilestoneTrackerBehavior(snapshotProvider.getInitialSnapshot().getIndex() + 1, Hash.NULL_HASH); - EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, latestMilestoneTracker); + EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, milestoneSolidifier); Hash entryPoint = entryPointSelector.getEntryPoint(10); Assert.assertEquals("The entry point should be the milestone in the Tangle", milestoneHash, entryPoint); @@ -54,7 +54,7 @@ public void testEntryPointBWithTangleData() throws Exception { public void testEntryPointAWithoutTangleData() throws Exception { mockMilestoneTrackerBehavior(0, Hash.NULL_HASH); - EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, latestMilestoneTracker); + EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, milestoneSolidifier); Hash entryPoint = entryPointSelector.getEntryPoint(10); Assert.assertEquals("The entry point should be the last tracked solid milestone", Hash.NULL_HASH, entryPoint); @@ -64,7 +64,7 @@ public void testEntryPointAWithoutTangleData() throws Exception { private void mockMilestoneTrackerBehavior(int latestSolidSubtangleMilestoneIndex, Hash latestSolidSubtangleMilestone) { snapshotProvider.getLatestSnapshot().setIndex(latestSolidSubtangleMilestoneIndex); snapshotProvider.getLatestSnapshot().setHash(latestSolidSubtangleMilestone); - Mockito.when(latestMilestoneTracker.getLatestMilestoneIndex()).thenReturn(latestSolidSubtangleMilestoneIndex); + Mockito.when(milestoneSolidifier.getLatestMilestoneIndex()).thenReturn(latestSolidSubtangleMilestoneIndex); } private void mockTangleBehavior(Hash milestoneModelHash) throws Exception { diff --git a/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java b/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java new file mode 100644 index 0000000000..d27ca7164c --- /dev/null +++ b/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java @@ -0,0 +1,102 @@ +package com.iota.iri.service.validation; + +import com.iota.iri.conf.MainnetConfig; +import com.iota.iri.conf.ProtocolConfig; +import com.iota.iri.controllers.TipsViewModel; +import com.iota.iri.crypto.SpongeFactory; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; +import com.iota.iri.storage.Tangle; +import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; +import com.iota.iri.utils.Converter; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.Rule; +import org.junit.AfterClass; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import static com.iota.iri.TransactionTestUtils.getTransactionTrits; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransactionValidatorTest { + + private static final int MAINNET_MWM = 14; + private static final TemporaryFolder dbFolder = new TemporaryFolder(); + private static final TemporaryFolder logFolder = new TemporaryFolder(); + private static Tangle tangle; + private static TransactionValidator txValidator; + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private static SnapshotProvider snapshotProvider; + + @Mock + private static TransactionRequester txRequester; + + @BeforeClass + public static void setUp() throws Exception { + dbFolder.create(); + logFolder.create(); + tangle = new Tangle(); + tangle.addPersistenceProvider( + new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); + tangle.init(); + } + + @AfterClass + public static void tearDown() throws Exception { + tangle.shutdown(); + dbFolder.delete(); + logFolder.delete(); + } + + @Before + public void setUpEach() { + when(snapshotProvider.getInitialSnapshot()).thenReturn(SnapshotMockUtils.createSnapshot()); + TipsViewModel tipsViewModel = new TipsViewModel(); + txRequester = new TransactionRequester(tangle, snapshotProvider); + txValidator = new TransactionValidator(snapshotProvider, txRequester, new MainnetConfig()); + txValidator.setMwm(false, MAINNET_MWM); + } + + @Test + public void testMinMwm() { + ProtocolConfig protocolConfig = mock(ProtocolConfig.class); + when(protocolConfig.getMwm()).thenReturn(5); + TransactionValidator transactionValidator = new TransactionValidator(null, null, protocolConfig); + assertEquals("Expected testnet minimum minWeightMagnitude", 13, transactionValidator.getMinWeightMagnitude()); + } + + @Test + public void validateTrits() { + byte[] trits = getTransactionTrits(); + Converter.copyTrits(0, trits, 0, trits.length); + txValidator.validateTrits(trits, MAINNET_MWM); + } + + @Test(expected = RuntimeException.class) + public void validateTritsWithInvalidMetadata() { + byte[] trits = getTransactionTrits(); + txValidator.validateTrits(trits, MAINNET_MWM); + } + + @Test + public void validateBytesWithNewCurl() { + byte[] trits = getTransactionTrits(); + Converter.copyTrits(0, trits, 0, trits.length); + byte[] bytes = Converter.allocateBytesForTrits(trits.length); + Converter.bytes(trits, 0, bytes, 0, trits.length); + txValidator.validateBytes(bytes, txValidator.getMinWeightMagnitude(), SpongeFactory.create(SpongeFactory.Mode.CURLP81)); + } +} diff --git a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java new file mode 100644 index 0000000000..c8d7d1af3d --- /dev/null +++ b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java @@ -0,0 +1,163 @@ +package com.iota.iri.service.validation.impl; + +import com.iota.iri.conf.ConfigFactory; +import com.iota.iri.conf.IotaConfig; +import com.iota.iri.controllers.TipsViewModel; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.crypto.SpongeFactory; +import com.iota.iri.model.TransactionHash; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; +import com.iota.iri.storage.Tangle; +import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; +import com.iota.iri.utils.Converter; +import org.junit.Rule; +import org.junit.BeforeClass; +import org.junit.Before; +import org.junit.AfterClass; +import org.junit.After; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import static com.iota.iri.TransactionTestUtils.getTransactionTrits; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +public class TransactionSolidifierImplTest { + private static final TemporaryFolder dbFolder = new TemporaryFolder(); + private static final TemporaryFolder logFolder = new TemporaryFolder(); + private static Tangle tangle; + private IotaConfig config = ConfigFactory.createIotaConfig(true); + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private static SnapshotProvider snapshotProvider; + + @Mock + private static TransactionSolidifierImpl txSolidifier; + + @Mock + private static TipsViewModel tipsViewModel; + + @Mock + private static TransactionRequester txRequester; + + @BeforeClass + public static void setUp() throws Exception { + dbFolder.create(); + logFolder.create(); + tangle = new Tangle(); + tangle.addPersistenceProvider( + new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); + tangle.init(); + } + + @AfterClass + public static void tearDown() throws Exception { + tangle.shutdown(); + dbFolder.delete(); + logFolder.delete(); + } + + @Before + public void setUpEach() { + when(snapshotProvider.getInitialSnapshot()).thenReturn(SnapshotMockUtils.createSnapshot()); + txRequester = new TransactionRequester(tangle, snapshotProvider); + txSolidifier = new TransactionSolidifierImpl(tangle, snapshotProvider, txRequester, tipsViewModel, config.getCoordinator()); + txSolidifier.start(); + } + + @After + public void tearDownEach(){ + txSolidifier.shutdown(); + } + + + @Test + public void verifyTxIsSolid() throws Exception { + TransactionViewModel tx = getTxWithBranchAndTrunk(); + assertTrue("Expected transaction to be solid", txSolidifier.checkSolidity(tx.getHash())); + assertTrue("Expected transaction to be solid", txSolidifier.checkSolidity(tx.getHash())); + } + + @Test + public void verifyTxIsNotSolid() throws Exception { + TransactionViewModel tx = getTxWithoutBranchAndTrunk(); + assertFalse("Expected transaction to fail solidity check", txSolidifier.checkSolidity(tx.getHash())); + assertFalse("Expected transaction to fail solidity check", txSolidifier.checkSolidity(tx.getHash())); + } + + @Test + public void getSolidificationQueue() throws Exception { + TransactionViewModel mainTx = getTxWithBranchAndTrunk(); + for(int i = 0; i < 10; i++) { + TransactionViewModel tx = getTxWithBranchAndTrunk(); + txSolidifier.addToSolidificationQueue(tx.getHash()); + } + txSolidifier.addToSolidificationQueue(mainTx.getHash()); + assertTrue("Expected transaction to be present in the solidification queue", + txSolidifier.getSolidificationQueue().contains(mainTx.getHash())); + } + + @Test + public void verifyTransactionIsProcessedFully() throws Exception { + TransactionViewModel tx = getTxWithBranchAndTrunk(); + txSolidifier.addToSolidificationQueue(tx.getHash()); + + //Time to process through the steps + Thread.sleep(1000); + assertTrue("Expected transaction to be present in the broadcast queue", + txSolidifier.getBroadcastQueue().contains(tx)); + } + + + @Test + public void verifyInconsistentTransactionIsNotProcessedFully() throws Exception { + TransactionViewModel tx = getTxWithoutBranchAndTrunk(); + txSolidifier.addToSolidificationQueue(tx.getHash()); + + //Time to process through the steps + Thread.sleep(1000); + assertFalse("Expected transaction not to be present in the broadcast queue", + txSolidifier.getBroadcastQueue().contains(tx)); + } + + private TransactionViewModel getTxWithBranchAndTrunk() throws Exception { + TransactionViewModel tx, trunkTx, branchTx; + String trytes = "999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999CFDEZBLZQYA9999999999999999999999999999999999999999999ZZWQHWD99C99999999C99999999CKWWDBWSCLMQULCTAAJGXDEMFJXPMGMAQIHDGHRBGEMUYNNCOK9YPHKEEFLFCZUSPMCJHAKLCIBQSGWAS999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999"; + + byte[] trits = Converter.allocateTritsForTrytes(trytes.length()); + Converter.trits(trytes, trits, 0); + trunkTx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); + branchTx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); + + byte[] childTx = getTransactionTrits(); + System.arraycopy(trunkTx.getHash().trits(), 0, childTx, TransactionViewModel.TRUNK_TRANSACTION_TRINARY_OFFSET, TransactionViewModel.TRUNK_TRANSACTION_TRINARY_SIZE); + System.arraycopy(branchTx.getHash().trits(), 0, childTx, TransactionViewModel.BRANCH_TRANSACTION_TRINARY_OFFSET, TransactionViewModel.BRANCH_TRANSACTION_TRINARY_SIZE); + tx = new TransactionViewModel(childTx, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, childTx)); + + trunkTx.store(tangle, snapshotProvider.getInitialSnapshot()); + branchTx.store(tangle, snapshotProvider.getInitialSnapshot()); + tx.store(tangle, snapshotProvider.getInitialSnapshot()); + + return tx; + } + + private TransactionViewModel getTxWithoutBranchAndTrunk() throws Exception { + byte[] trits = getTransactionTrits(); + TransactionViewModel tx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); + + tx.store(tangle, snapshotProvider.getInitialSnapshot()); + + return tx; + } + +}