From 53ea16607384b26c82e660a6ce384ed5ee1dc910 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 14 Oct 2019 13:02:03 -0600 Subject: [PATCH 01/48] Add ubiquitous Broadcast Queue --- src/main/java/com/iota/iri/IRI.java | 3 ++ src/main/java/com/iota/iri/Iota.java | 13 +++++- .../com/iota/iri/TransactionValidator.java | 8 ++-- .../com/iota/iri/conf/BaseIotaConfig.java | 8 ++++ .../java/com/iota/iri/conf/NetworkConfig.java | 7 +++ .../iri/controllers/TransactionViewModel.java | 8 +++- .../NetworkInjectionConfiguration.java | 3 +- .../iri/network/pipeline/BroadcastQueue.java | 46 +++++++++++++++++++ .../TransactionProcessingPipelineImpl.java | 21 ++++++--- 9 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/iota/iri/network/pipeline/BroadcastQueue.java diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index cb03cc19c2..6e02230b3c 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -7,6 +7,7 @@ import com.iota.iri.conf.ConfigFactory; import com.iota.iri.conf.IotaConfig; import com.iota.iri.network.NetworkInjectionConfiguration; +import com.iota.iri.network.pipeline.BroadcastQueue; import com.iota.iri.service.API; import com.iota.iri.utils.IotaUtils; import com.iota.iri.service.restserver.resteasy.RestEasy; @@ -119,6 +120,7 @@ private static class IRILauncher { public static void main(String [] args) throws Exception { IotaConfig config = createConfiguration(args); String version = IotaUtils.getIriVersion(); + BroadcastQueue broadcastQueue = config.getBroadcastQueue(); log.info("Welcome to {} {}", config.isTestnet() ? TESTNET_NAME : MAINNET_NAME, version); Injector injector = Guice.createInjector( @@ -132,6 +134,7 @@ public static void main(String [] args) throws Exception { shutdownHook(); try { + iota.configureBroadcastQueue(broadcastQueue); iota.init(); //TODO redundant parameter but we will touch this when we refactor IXI ixi.init(config.getIxiDir()); diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index a996a93e17..6336593b56 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -6,6 +6,7 @@ import com.iota.iri.network.NeighborRouter; import com.iota.iri.network.TipsRequester; import com.iota.iri.network.TransactionRequester; +import com.iota.iri.network.pipeline.BroadcastQueue; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.ledger.LedgerService; import com.iota.iri.service.milestone.*; @@ -106,6 +107,8 @@ public class Iota { public final TipsViewModel tipsViewModel; public final TipSelector tipsSelector; + private BroadcastQueue broadcastQueue; + /** * Initializes the latest snapshot and then creates all services needed to run an IOTA node. * @@ -181,7 +184,7 @@ public void init() throws Exception { tangle.clearMetadata(com.iota.iri.model.persistables.Transaction.class); } - transactionValidator.init(); + transactionValidator.init(this.broadcastQueue); txPipeline.start(); neighborRouter.start(); @@ -274,6 +277,14 @@ private void initializeTangle() { } } + /** + * Sets the {@link BroadcastQueue} for this IRI instance + * @param broadcastQueue Baseline IRI {@link BroadcastQueue} + */ + public void configureBroadcastQueue(BroadcastQueue broadcastQueue){ + this.broadcastQueue = broadcastQueue; + } + /** * Creates a new Persistable provider with the supplied settings * diff --git a/src/main/java/com/iota/iri/TransactionValidator.java b/src/main/java/com/iota/iri/TransactionValidator.java index 741a3816d7..a0ee3b01e2 100644 --- a/src/main/java/com/iota/iri/TransactionValidator.java +++ b/src/main/java/com/iota/iri/TransactionValidator.java @@ -10,6 +10,7 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; +import com.iota.iri.network.pipeline.BroadcastQueue; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; import org.slf4j.Logger; @@ -33,7 +34,7 @@ public class TransactionValidator { private static final long MAX_TIMESTAMP_FUTURE = 2L * 60L * 60L; private static final long MAX_TIMESTAMP_FUTURE_MS = MAX_TIMESTAMP_FUTURE * 1_000L; - + private BroadcastQueue broadcastQueue; /////////////////////////////////fields for solidification thread////////////////////////////////////// private Thread newSolidThread; @@ -85,7 +86,8 @@ public class TransactionValidator { * * @see #spawnSolidTransactionsPropagation() */ - public void init() { + public void init(BroadcastQueue broadcastQueue) { + this.broadcastQueue = broadcastQueue; newSolidThread.start(); } @@ -277,7 +279,7 @@ public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exc } } if (solid) { - updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), analyzedHashes); + updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), analyzedHashes, broadcastQueue); } analyzedHashes.clear(); return solid; diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index dc54fefe35..b8adb4dc14 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -4,12 +4,14 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; import com.iota.iri.utils.IotaUtils; +import com.iota.iri.network.pipeline.BroadcastQueue; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import java.util.concurrent.BlockingQueue; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -56,6 +58,7 @@ public abstract class BaseIotaConfig implements IotaConfig { protected boolean dnsRefresherEnabled = Defaults.DNS_REFRESHER_ENABLED; protected boolean dnsResolutionEnabled = Defaults.DNS_RESOLUTION_ENABLED; protected List neighbors = Collections.EMPTY_LIST; + protected BroadcastQueue broadcastQueue = new BroadcastQueue(); //IXI protected String ixiDir = Defaults.IXI_DIR; @@ -169,6 +172,11 @@ public String getApiHost() { return apiHost; } + @Override + public BroadcastQueue getBroadcastQueue() { + return broadcastQueue; + } + @JsonProperty @Parameter(names = {"--api-host"}, description = APIConfig.Descriptions.API_HOST) protected void setApiHost(String apiHost) { diff --git a/src/main/java/com/iota/iri/conf/NetworkConfig.java b/src/main/java/com/iota/iri/conf/NetworkConfig.java index 56b1f33949..a933128759 100644 --- a/src/main/java/com/iota/iri/conf/NetworkConfig.java +++ b/src/main/java/com/iota/iri/conf/NetworkConfig.java @@ -1,5 +1,7 @@ package com.iota.iri.conf; +import com.iota.iri.network.pipeline.BroadcastQueue; + import java.util.List; /** @@ -89,6 +91,11 @@ public interface NetworkConfig extends Config { */ int getCacheSizeBytes(); + /** + * @return The current {@link BroadcastQueue} + */ + BroadcastQueue getBroadcastQueue(); + interface Descriptions { String NEIGHBORING_SOCKET_ADDRESS = "The address to bind the TCP server socket to."; String NEIGHBORING_SOCKET_PORT = "The TCP Receiver Port."; diff --git a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java index 50668790a9..f645317b92 100644 --- a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java +++ b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java @@ -2,6 +2,9 @@ import com.iota.iri.model.*; import com.iota.iri.model.persistables.*; +import com.iota.iri.network.pipeline.BroadcastQueue; +import com.iota.iri.network.pipeline.ProcessingContext; +import com.iota.iri.network.pipeline.BroadcastPayload; import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; @@ -712,7 +715,8 @@ public void setMetadata() { : TransactionViewModel.FILLED_SLOT; } - public static void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, final LinkedHashSet analyzedHashes) + public static void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, + final LinkedHashSet analyzedHashes, BroadcastQueue broadcastQueue) throws Exception { Object[] hashes = analyzedHashes.toArray(); TransactionViewModel transactionViewModel; @@ -724,6 +728,8 @@ public static void updateSolidTransactions(Tangle tangle, Snapshot initialSnapsh if (!transactionViewModel.isSolid()) { transactionViewModel.updateSolid(true); transactionViewModel.update(tangle, initialSnapshot, "solid|height"); + BroadcastPayload payload = new BroadcastPayload(null,transactionViewModel); + broadcastQueue.add(new ProcessingContext(payload)); } } } diff --git a/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java b/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java index 032715736d..b60bad3f80 100644 --- a/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java @@ -48,7 +48,8 @@ TransactionProcessingPipeline provideTransactionProcessingPipeline(NeighborRoute TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, TransactionRequester transactionRequester) { return new TransactionProcessingPipelineImpl(neighborRouter, configuration, txValidator, tangle, - snapshotProvider, tipsViewModel, latestMilestoneTracker, transactionRequester); + snapshotProvider, tipsViewModel, latestMilestoneTracker, transactionRequester, + configuration.getBroadcastQueue()); } @Singleton diff --git a/src/main/java/com/iota/iri/network/pipeline/BroadcastQueue.java b/src/main/java/com/iota/iri/network/pipeline/BroadcastQueue.java new file mode 100644 index 0000000000..74c8774be9 --- /dev/null +++ b/src/main/java/com/iota/iri/network/pipeline/BroadcastQueue.java @@ -0,0 +1,46 @@ +package com.iota.iri.network.pipeline; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +/** + * A queue for transactions intended to be submitted to the {@link BroadcastStage} + * for processing + */ +public class BroadcastQueue { + + /** A blocking queue to store transactions for broadcasting */ + private BlockingQueue broadcastStageQueue = new ArrayBlockingQueue<>(100); + + /** An object to be used for synchronizing calls */ + private final Object broadcastSync = new Object(); + + + /** + * Add transactions to the Broadcast Queue + * @param context Transaction context to be passed to the {@link BroadcastStage} + * @return True if added properly, False if not + */ + public boolean add(ProcessingContext context) { + synchronized (broadcastSync) { + try { + this.broadcastStageQueue.put(context); + return true; + } catch (Exception e) { + return false; + } + } + + } + + /** + * Getter for the current Broadcast Queue + * @return BlockingQueue of all transactions left to be broadcasted + */ + public BlockingQueue get(){ + /** Call is synchronized to ensure all pending additions have completed before sending the state */ + synchronized (broadcastSync) { + return this.broadcastStageQueue; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index 5c49300f5a..b2bbb22744 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -66,8 +66,8 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP private BlockingQueue preProcessStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue validationStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue receivedStageQueue = new ArrayBlockingQueue<>(100); - private BlockingQueue broadcastStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue replyStageQueue = new ArrayBlockingQueue<>(100); + private BroadcastQueue broadcastStageQueue; /** * Creates a {@link TransactionProcessingPipeline}. @@ -84,7 +84,7 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP public TransactionProcessingPipelineImpl(NeighborRouter neighborRouter, NodeConfig config, TransactionValidator txValidator, Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, - TransactionRequester transactionRequester) { + TransactionRequester transactionRequester, BroadcastQueue broadcastStageQueue) { FIFOCache recentlySeenBytesCache = new FIFOCache<>(config.getCacheSizeBytes()); this.preProcessStage = new PreProcessStage(recentlySeenBytesCache); this.replyStage = new ReplyStage(neighborRouter, config, tangle, tipsViewModel, latestMilestoneTracker, @@ -94,6 +94,7 @@ public TransactionProcessingPipelineImpl(NeighborRouter neighborRouter, NodeConf this.receivedStage = new ReceivedStage(tangle, txValidator, snapshotProvider, transactionRequester); this.batchedHasher = BatchedHasherFactory.create(BatchedHasherFactory.Type.BCTCURL81, 20); this.hashingStage = new HashingStage(batchedHasher); + this.broadcastStageQueue = broadcastStageQueue; } @Override @@ -103,7 +104,7 @@ public void start() { addStage("validation", validationStageQueue, validationStage); addStage("reply", replyStageQueue, replyStage); addStage("received", receivedStageQueue, receivedStage); - addStage("broadcast", broadcastStageQueue, broadcastStage); + addStage("broadcast", broadcastStageQueue.get(), broadcastStage); } /** @@ -118,7 +119,15 @@ private void addStage(String name, BlockingQueue queue, stagesThreadPool.submit(new Thread(() -> { try { while (!Thread.currentThread().isInterrupted()) { - ProcessingContext ctx = stage.process(queue.take()); + ProcessingContext queueTake; + if(name.equals("broadcast")) { + log.info("Broadcasting"); + queueTake = broadcastStageQueue.get().take(); + } else{ + queueTake = queue.take(); + } + ProcessingContext ctx = stage.process(queueTake); + switch (ctx.getNextStage()) { case REPLY: replyStageQueue.put(ctx); @@ -135,7 +144,7 @@ private void addStage(String name, BlockingQueue queue, receivedStageQueue.put(payload.getRight()); break; case BROADCAST: - broadcastStageQueue.put(ctx); + broadcastStageQueue.add(ctx); break; case ABORT: break; @@ -160,7 +169,7 @@ public BlockingQueue getReceivedStageQueue() { @Override public BlockingQueue getBroadcastStageQueue() { - return broadcastStageQueue; + return broadcastStageQueue.get(); } @Override From be7c132c6ab3a982dca047f1a2ee3a384c81e3b5 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Tue, 15 Oct 2019 01:05:57 -0600 Subject: [PATCH 02/48] Update unit tests with broadcasting queue --- .../iota/iri/TransactionValidatorTest.java | 5 +++ .../TransactionProcessingPipelineTest.java | 31 ++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java index b1fc48f980..cdc361d441 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/TransactionValidatorTest.java @@ -7,6 +7,7 @@ import com.iota.iri.crypto.SpongeFactory; import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; +import com.iota.iri.network.pipeline.BroadcastQueue; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; import com.iota.iri.storage.Tangle; @@ -42,6 +43,9 @@ public class TransactionValidatorTest { @Mock private static SnapshotProvider snapshotProvider; + @Mock + private static BroadcastQueue broadcastQueue; + @BeforeClass public static void setUp() throws Exception { dbFolder.create(); @@ -67,6 +71,7 @@ public void setUpEach() { TransactionRequester txRequester = new TransactionRequester(tangle, snapshotProvider); txValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, txRequester, new MainnetConfig()); txValidator.setMwm(false, MAINNET_MWM); + txValidator.init(broadcastQueue); } @Test diff --git a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java index b9b8c4ab8f..58d4a356cb 100644 --- a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java @@ -11,6 +11,7 @@ import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; +import com.sun.jmx.remote.internal.ArrayQueue; import org.junit.Rule; import org.junit.Test; import org.mockito.Mock; @@ -18,6 +19,11 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; +import java.lang.reflect.Array; +import java.util.Queue; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + public class TransactionProcessingPipelineTest { @Rule @@ -86,12 +92,21 @@ public class TransactionProcessingPipelineTest { @Mock private ProcessingContext receivedCtx; + @Mock + private ProcessingContext finishCtx; + @Mock private ProcessingContext broadcastCtx; @Mock private ProcessingContext abortCtx; + @Mock + private BroadcastQueue broadcastQueue; + + @Mock + private BlockingQueue blockingQueue; + private void mockHashingStage(TransactionProcessingPipeline pipeline) { Mockito.when(hashingPayload.getTxTrits()).thenReturn(null); Mockito.doAnswer(invocation -> { @@ -114,7 +129,7 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester); + transactionRequester, broadcastQueue); // inject mocks injectMockedStagesIntoPipeline(pipeline); @@ -138,6 +153,10 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); Mockito.when(receivedStage.process(receivedCtx)).thenReturn(broadcastCtx); + // mock broadcastQueue + Mockito.when(broadcastQueue.get()).thenReturn(blockingQueue); + Mockito.when(blockingQueue.take()).thenReturn(broadcastCtx); + pipeline.start(); // send in actual payload to kick off the 'processing' @@ -159,7 +178,7 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws public void processingAKnownTransactionOnlyFlowsToTheReplyStage() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester); + transactionRequester, broadcastQueue); // inject mocks pipeline.setPreProcessStage(preProcessStage); @@ -193,7 +212,7 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester); + transactionRequester, broadcastQueue); // inject mocks injectMockedStagesIntoPipeline(pipeline); @@ -214,6 +233,10 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); Mockito.when(receivedStage.process(receivedCtx)).thenReturn(broadcastCtx); + // mock broadcastQueue + Mockito.when(broadcastQueue.get()).thenReturn(blockingQueue); + Mockito.when(blockingQueue.take()).thenReturn(broadcastCtx); + pipeline.start(); // send into the pipeline but without originating from a neighbor @@ -238,7 +261,7 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug public void anInvalidNewTransactionStopsBeingProcessedAfterTheValidationStage() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester); + transactionRequester, broadcastQueue); // inject mocks injectMockedStagesIntoPipeline(pipeline); From 6593388893feb8f68bd537f3183f11ffbf6053ee Mon Sep 17 00:00:00 2001 From: DyrellC Date: Thu, 17 Oct 2019 11:46:56 -0600 Subject: [PATCH 03/48] Remove unneeded log --- .../iri/network/pipeline/TransactionProcessingPipelineImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index b2bbb22744..f30804477b 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -121,7 +121,6 @@ private void addStage(String name, BlockingQueue queue, while (!Thread.currentThread().isInterrupted()) { ProcessingContext queueTake; if(name.equals("broadcast")) { - log.info("Broadcasting"); queueTake = broadcastStageQueue.get().take(); } else{ queueTake = queue.take(); From ff5849534c8643f46dda5387ef2c0fb88388859d Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 4 Nov 2019 07:30:05 -0700 Subject: [PATCH 04/48] Fix broadcast queue injection --- src/main/java/com/iota/iri/IRI.java | 2 -- src/main/java/com/iota/iri/MainInjectionConfiguration.java | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index 6e02230b3c..ce7fd96660 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -120,7 +120,6 @@ private static class IRILauncher { public static void main(String [] args) throws Exception { IotaConfig config = createConfiguration(args); String version = IotaUtils.getIriVersion(); - BroadcastQueue broadcastQueue = config.getBroadcastQueue(); log.info("Welcome to {} {}", config.isTestnet() ? TESTNET_NAME : MAINNET_NAME, version); Injector injector = Guice.createInjector( @@ -134,7 +133,6 @@ public static void main(String [] args) throws Exception { shutdownHook(); try { - iota.configureBroadcastQueue(broadcastQueue); iota.init(); //TODO redundant parameter but we will touch this when we refactor IXI ixi.init(config.getIxiDir()); diff --git a/src/main/java/com/iota/iri/MainInjectionConfiguration.java b/src/main/java/com/iota/iri/MainInjectionConfiguration.java index dc2075b710..3a4d999fb4 100644 --- a/src/main/java/com/iota/iri/MainInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/MainInjectionConfiguration.java @@ -9,6 +9,7 @@ import com.iota.iri.network.NeighborRouter; import com.iota.iri.network.TipsRequester; import com.iota.iri.network.TransactionRequester; +import com.iota.iri.network.pipeline.BroadcastQueue; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.API; import com.iota.iri.service.ledger.LedgerService; @@ -204,4 +205,10 @@ protected void configure() { bind(TipsViewModel.class).asEagerSingleton(); } + @Singleton + @Provides + BroadcastQueue provideBroadcastQueue(){ + return configuration.getBroadcastQueue(); + } + } From 1a0ea439343706f253426e5db0504ae215845c24 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 4 Nov 2019 07:30:47 -0700 Subject: [PATCH 05/48] remove unused function --- src/main/java/com/iota/iri/Iota.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 6336593b56..0fbdffa066 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -277,14 +277,6 @@ private void initializeTangle() { } } - /** - * Sets the {@link BroadcastQueue} for this IRI instance - * @param broadcastQueue Baseline IRI {@link BroadcastQueue} - */ - public void configureBroadcastQueue(BroadcastQueue broadcastQueue){ - this.broadcastQueue = broadcastQueue; - } - /** * Creates a new Persistable provider with the supplied settings * From 24e31a57e96c63e857764407feb901ca18dcae4e Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 4 Nov 2019 08:37:17 -0700 Subject: [PATCH 06/48] Clean up imports --- src/main/java/com/iota/iri/conf/BaseIotaConfig.java | 1 - .../com/iota/iri/controllers/TransactionViewModel.java | 2 +- .../pipeline/TransactionProcessingPipelineTest.java | 7 ------- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index 2675862e41..82de999aea 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -11,7 +11,6 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import java.util.concurrent.BlockingQueue; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; diff --git a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java index f645317b92..d2a83e2608 100644 --- a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java +++ b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java @@ -716,7 +716,7 @@ public void setMetadata() { } public static void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, - final LinkedHashSet analyzedHashes, BroadcastQueue broadcastQueue) + final Set analyzedHashes, BroadcastQueue broadcastQueue) throws Exception { Object[] hashes = analyzedHashes.toArray(); TransactionViewModel transactionViewModel; diff --git a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java index 58d4a356cb..40d348ae1d 100644 --- a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java @@ -11,7 +11,6 @@ import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; -import com.sun.jmx.remote.internal.ArrayQueue; import org.junit.Rule; import org.junit.Test; import org.mockito.Mock; @@ -19,9 +18,6 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -import java.lang.reflect.Array; -import java.util.Queue; -import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; public class TransactionProcessingPipelineTest { @@ -92,9 +88,6 @@ public class TransactionProcessingPipelineTest { @Mock private ProcessingContext receivedCtx; - @Mock - private ProcessingContext finishCtx; - @Mock private ProcessingContext broadcastCtx; From 03590dd9da3deed4fa686ce7cc23bc3e094db0e3 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Fri, 8 Nov 2019 01:02:11 -0700 Subject: [PATCH 07/48] Remove BroadcastQueue, add TransactionSolidifier class --- src/main/java/com/iota/iri/IRI.java | 1 - src/main/java/com/iota/iri/Iota.java | 14 +- .../iota/iri/MainInjectionConfiguration.java | 27 ++- .../com/iota/iri/conf/BaseIotaConfig.java | 7 - .../java/com/iota/iri/conf/NetworkConfig.java | 7 - .../iri/controllers/TransactionViewModel.java | 24 +- .../NetworkInjectionConfiguration.java | 8 +- .../iri/network/pipeline/BroadcastQueue.java | 46 ---- .../iri/network/pipeline/ReceivedStage.java | 2 +- .../TransactionProcessingPipeline.java | 7 + .../TransactionProcessingPipelineImpl.java | 49 ++-- .../iri/network/pipeline/ValidationStage.java | 2 +- src/main/java/com/iota/iri/service/API.java | 2 +- .../impl/MilestoneSolidifierImpl.java | 18 +- .../validation/TransactionSolidifier.java | 223 ++++++++++++++++++ .../validation}/TransactionValidator.java | 94 +------- .../iri/MainInjectionConfigurationTest.java | 1 + .../iota/iri/TransactionValidatorTest.java | 24 +- .../NetworkInjectionConfigurationTest.java | 2 +- .../network/pipeline/ReceivedStageTest.java | 2 +- .../TransactionProcessingPipelineTest.java | 25 +- .../network/pipeline/ValidationStageTest.java | 2 +- .../java/com/iota/iri/service/APITest.java | 2 +- 23 files changed, 342 insertions(+), 247 deletions(-) delete mode 100644 src/main/java/com/iota/iri/network/pipeline/BroadcastQueue.java create mode 100644 src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java rename src/main/java/com/iota/iri/{ => service/validation}/TransactionValidator.java (79%) diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index ce7fd96660..cb03cc19c2 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -7,7 +7,6 @@ import com.iota.iri.conf.ConfigFactory; import com.iota.iri.conf.IotaConfig; import com.iota.iri.network.NetworkInjectionConfiguration; -import com.iota.iri.network.pipeline.BroadcastQueue; import com.iota.iri.service.API; import com.iota.iri.utils.IotaUtils; import com.iota.iri.service.restserver.resteasy.RestEasy; diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 0fbdffa066..b653ea3222 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -6,7 +6,6 @@ import com.iota.iri.network.NeighborRouter; import com.iota.iri.network.TipsRequester; import com.iota.iri.network.TransactionRequester; -import com.iota.iri.network.pipeline.BroadcastQueue; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.ledger.LedgerService; import com.iota.iri.service.milestone.*; @@ -19,6 +18,8 @@ import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.tipselection.TipSelector; import com.iota.iri.service.transactionpruning.TransactionPruner; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.PersistenceProvider; @@ -95,6 +96,8 @@ public class Iota { public final MilestoneSolidifier milestoneSolidifier; + public final TransactionSolidifier transactionSolidifier; + public final BundleValidator bundleValidator; public final Tangle tangle; @@ -107,15 +110,13 @@ public class Iota { public final TipsViewModel tipsViewModel; public final TipSelector tipsSelector; - private BroadcastQueue broadcastQueue; - /** * Initializes the latest snapshot and then creates all services needed to run an IOTA node. * * @param configuration Information about how this node will be configured. * */ - public Iota(IotaConfig configuration, SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, LocalSnapshotManager localSnapshotManager, MilestoneService milestoneService, LatestMilestoneTracker latestMilestoneTracker, LatestSolidMilestoneTracker latestSolidMilestoneTracker, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector) { + public Iota(IotaConfig configuration, SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, LocalSnapshotManager localSnapshotManager, MilestoneService milestoneService, LatestMilestoneTracker latestMilestoneTracker, LatestSolidMilestoneTracker latestSolidMilestoneTracker, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector, TransactionSolidifier transactionSolidifier) { this.configuration = configuration; this.ledgerService = ledgerService; @@ -133,6 +134,7 @@ public Iota(IotaConfig configuration, SpentAddressesProvider spentAddressesProvi this.neighborRouter = neighborRouter; this.txPipeline = transactionProcessingPipeline; this.tipsRequester = tipsRequester; + this.transactionSolidifier = transactionSolidifier; // legacy classes this.bundleValidator = bundleValidator; @@ -184,7 +186,7 @@ public void init() throws Exception { tangle.clearMetadata(com.iota.iri.model.persistables.Transaction.class); } - transactionValidator.init(this.broadcastQueue); + transactionValidator.init(); txPipeline.start(); neighborRouter.start(); @@ -194,6 +196,7 @@ public void init() throws Exception { latestSolidMilestoneTracker.start(); seenMilestonesRetriever.start(); milestoneSolidifier.start(); + transactionSolidifier.start(); if (localSnapshotManager != null) { localSnapshotManager.start(latestMilestoneTracker); @@ -235,6 +238,7 @@ private void rescanDb() throws Exception { public void shutdown() throws Exception { // shutdown in reverse starting order (to not break any dependencies) milestoneSolidifier.shutdown(); + transactionSolidifier.shutdown(); seenMilestonesRetriever.shutdown(); latestSolidMilestoneTracker.shutdown(); latestMilestoneTracker.shutdown(); diff --git a/src/main/java/com/iota/iri/MainInjectionConfiguration.java b/src/main/java/com/iota/iri/MainInjectionConfiguration.java index 4510efd8f5..5699addc3c 100644 --- a/src/main/java/com/iota/iri/MainInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/MainInjectionConfiguration.java @@ -9,7 +9,6 @@ import com.iota.iri.network.NeighborRouter; import com.iota.iri.network.TipsRequester; import com.iota.iri.network.TransactionRequester; -import com.iota.iri.network.pipeline.BroadcastQueue; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.API; import com.iota.iri.service.ledger.LedgerService; @@ -46,6 +45,8 @@ import com.iota.iri.service.tipselection.impl.WalkerAlpha; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.PersistenceProvider; import com.iota.iri.storage.Tangle; @@ -136,8 +137,8 @@ SeenMilestonesRetriever provideSeenMilestonesRetriever(Tangle tangle, SnapshotPr @Singleton @Provides - MilestoneSolidifier provideMilestoneSolidifier(SnapshotProvider snapshotProvider, TransactionValidator transactionValidator) { - return new MilestoneSolidifierImpl(snapshotProvider, transactionValidator); + MilestoneSolidifier provideMilestoneSolidifier(SnapshotProvider snapshotProvider, TransactionSolidifier transactionSolidifier) { + return new MilestoneSolidifierImpl(snapshotProvider, transactionSolidifier); } @Singleton @@ -158,8 +159,14 @@ LocalSnapshotManager provideLocalSnapshotManager(SnapshotProvider snapshotProvid @Singleton @Provides - TransactionValidator provideTransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester) { - return new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester, configuration); + TransactionValidator provideTransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, TransactionSolidifier transactionSolidifier) { + return new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester, configuration, transactionSolidifier); + } + + @Singleton + @Provides + TransactionSolidifier provideTransactionSolidifier(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester){ + return new TransactionSolidifier(tangle, snapshotProvider, transactionRequester); } @Singleton @@ -177,8 +184,8 @@ TipSelector provideTipSelector(Tangle tangle, SnapshotProvider snapshotProvider, @Singleton @Provides - Iota provideIota(SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, @Nullable LocalSnapshotManager localSnapshotManager, MilestoneService milestoneService, LatestMilestoneTracker latestMilestoneTracker, LatestSolidMilestoneTracker latestSolidMilestoneTracker, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, @Nullable TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector) { - return new Iota(configuration, spentAddressesProvider, spentAddressesService, snapshotProvider, snapshotService, localSnapshotManager, milestoneService, latestMilestoneTracker, latestSolidMilestoneTracker, seenMilestonesRetriever, ledgerService, transactionPruner, milestoneSolidifier, bundleValidator, tangle, transactionValidator, transactionRequester, neighborRouter, transactionProcessingPipeline, tipsRequester, tipsViewModel, tipsSelector); + Iota provideIota(SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, @Nullable LocalSnapshotManager localSnapshotManager, MilestoneService milestoneService, LatestMilestoneTracker latestMilestoneTracker, LatestSolidMilestoneTracker latestSolidMilestoneTracker, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, @Nullable TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector, TransactionSolidifier transactionSolidifier) { + return new Iota(configuration, spentAddressesProvider, spentAddressesService, snapshotProvider, snapshotService, localSnapshotManager, milestoneService, latestMilestoneTracker, latestSolidMilestoneTracker, seenMilestonesRetriever, ledgerService, transactionPruner, milestoneSolidifier, bundleValidator, tangle, transactionValidator, transactionRequester, neighborRouter, transactionProcessingPipeline, tipsRequester, tipsViewModel, tipsSelector, transactionSolidifier); } @Singleton @@ -205,10 +212,4 @@ protected void configure() { bind(TipsViewModel.class).asEagerSingleton(); } - @Singleton - @Provides - BroadcastQueue provideBroadcastQueue(){ - return configuration.getBroadcastQueue(); - } - } diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index 82de999aea..693c4db327 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -4,7 +4,6 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; import com.iota.iri.utils.IotaUtils; -import com.iota.iri.network.pipeline.BroadcastQueue; import java.net.InetAddress; import java.net.UnknownHostException; @@ -57,7 +56,6 @@ public abstract class BaseIotaConfig implements IotaConfig { protected boolean dnsRefresherEnabled = Defaults.DNS_REFRESHER_ENABLED; protected boolean dnsResolutionEnabled = Defaults.DNS_RESOLUTION_ENABLED; protected List neighbors = Collections.EMPTY_LIST; - protected BroadcastQueue broadcastQueue = new BroadcastQueue(); //IXI protected String ixiDir = Defaults.IXI_DIR; @@ -174,11 +172,6 @@ public String getApiHost() { return apiHost; } - @Override - public BroadcastQueue getBroadcastQueue() { - return broadcastQueue; - } - @JsonProperty @Parameter(names = {"--api-host"}, description = APIConfig.Descriptions.API_HOST) protected void setApiHost(String apiHost) { diff --git a/src/main/java/com/iota/iri/conf/NetworkConfig.java b/src/main/java/com/iota/iri/conf/NetworkConfig.java index a933128759..56b1f33949 100644 --- a/src/main/java/com/iota/iri/conf/NetworkConfig.java +++ b/src/main/java/com/iota/iri/conf/NetworkConfig.java @@ -1,7 +1,5 @@ package com.iota.iri.conf; -import com.iota.iri.network.pipeline.BroadcastQueue; - import java.util.List; /** @@ -91,11 +89,6 @@ public interface NetworkConfig extends Config { */ int getCacheSizeBytes(); - /** - * @return The current {@link BroadcastQueue} - */ - BroadcastQueue getBroadcastQueue(); - interface Descriptions { String NEIGHBORING_SOCKET_ADDRESS = "The address to bind the TCP server socket to."; String NEIGHBORING_SOCKET_PORT = "The TCP Receiver Port."; diff --git a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java index d2a83e2608..e504d90f73 100644 --- a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java +++ b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java @@ -2,10 +2,8 @@ import com.iota.iri.model.*; import com.iota.iri.model.persistables.*; -import com.iota.iri.network.pipeline.BroadcastQueue; -import com.iota.iri.network.pipeline.ProcessingContext; -import com.iota.iri.network.pipeline.BroadcastPayload; import com.iota.iri.service.snapshot.Snapshot; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.Tangle; @@ -715,29 +713,11 @@ public void setMetadata() { : TransactionViewModel.FILLED_SLOT; } - public static void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, - final Set analyzedHashes, BroadcastQueue broadcastQueue) - throws Exception { - Object[] hashes = analyzedHashes.toArray(); - TransactionViewModel transactionViewModel; - for(int i = hashes.length -1; i >= 0; i--){ - transactionViewModel = TransactionViewModel.fromHash(tangle, (Hash) hashes[i]); - - transactionViewModel.updateHeights(tangle, initialSnapshot); - - if (!transactionViewModel.isSolid()) { - transactionViewModel.updateSolid(true); - transactionViewModel.update(tangle, initialSnapshot, "solid|height"); - BroadcastPayload payload = new BroadcastPayload(null,transactionViewModel); - broadcastQueue.add(new ProcessingContext(payload)); - } - } - } /** * Updates the {@link Transaction#solid} value of the referenced {@link Transaction} object. * - * Used by the {@link com.iota.iri.TransactionValidator} to quickly set the solidity of a {@link Transaction} set. + * Used by the {@link TransactionValidator} to quickly set the solidity of a {@link Transaction} set. * * @param solid The solidity of the transaction in the database * @return True if the {@link Transaction#solid} has been updated, False if not. diff --git a/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java b/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java index b60bad3f80..d4caa618e5 100644 --- a/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java @@ -3,7 +3,8 @@ import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.IotaConfig; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.network.impl.TipsRequesterImpl; @@ -46,10 +47,9 @@ TipsRequester provideTipsRequester(NeighborRouter neighborRouter, Tangle tangle, TransactionProcessingPipeline provideTransactionProcessingPipeline(NeighborRouter neighborRouter, TransactionValidator txValidator, Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, - TransactionRequester transactionRequester) { + TransactionRequester transactionRequester, TransactionSolidifier transactionSolidifier) { return new TransactionProcessingPipelineImpl(neighborRouter, configuration, txValidator, tangle, - snapshotProvider, tipsViewModel, latestMilestoneTracker, transactionRequester, - configuration.getBroadcastQueue()); + snapshotProvider, tipsViewModel, latestMilestoneTracker, transactionRequester, transactionSolidifier); } @Singleton diff --git a/src/main/java/com/iota/iri/network/pipeline/BroadcastQueue.java b/src/main/java/com/iota/iri/network/pipeline/BroadcastQueue.java deleted file mode 100644 index 74c8774be9..0000000000 --- a/src/main/java/com/iota/iri/network/pipeline/BroadcastQueue.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.iota.iri.network.pipeline; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; - -/** - * A queue for transactions intended to be submitted to the {@link BroadcastStage} - * for processing - */ -public class BroadcastQueue { - - /** A blocking queue to store transactions for broadcasting */ - private BlockingQueue broadcastStageQueue = new ArrayBlockingQueue<>(100); - - /** An object to be used for synchronizing calls */ - private final Object broadcastSync = new Object(); - - - /** - * Add transactions to the Broadcast Queue - * @param context Transaction context to be passed to the {@link BroadcastStage} - * @return True if added properly, False if not - */ - public boolean add(ProcessingContext context) { - synchronized (broadcastSync) { - try { - this.broadcastStageQueue.put(context); - return true; - } catch (Exception e) { - return false; - } - } - - } - - /** - * Getter for the current Broadcast Queue - * @return BlockingQueue of all transactions left to be broadcasted - */ - public BlockingQueue get(){ - /** Call is synchronized to ensure all pending additions have completed before sending the state */ - synchronized (broadcastSync) { - return this.broadcastStageQueue; - } - } -} \ No newline at end of file diff --git a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java index e8c5e99711..2f07ce7b75 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java @@ -1,6 +1,6 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.Neighbor; diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java index 1f19e4248c..ec76dae82b 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java @@ -1,5 +1,6 @@ package com.iota.iri.network.pipeline; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.network.neighbor.Neighbor; import java.nio.ByteBuffer; @@ -65,6 +66,12 @@ enum Stage { */ void process(byte[] txTrits); + /** + * Fetches a set of transactions from the {@link com.iota.iri.service.validation.TransactionSolidifier} and submits + * the object into the {@link BroadcastStage} queue. + */ + void refillBroadcastQueue(); + /** * Shut downs the pipeline by shutting down all stages. */ diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index f30804477b..87cca53582 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -1,8 +1,10 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.NodeConfig; import com.iota.iri.controllers.TipsViewModel; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.crypto.batched.BatchedHasher; import com.iota.iri.crypto.batched.BatchedHasherFactory; import com.iota.iri.crypto.batched.HashRequest; @@ -19,7 +21,10 @@ import com.iota.iri.utils.Converter; import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; @@ -62,12 +67,13 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP private BroadcastStage broadcastStage; private BatchedHasher batchedHasher; private HashingStage hashingStage; + private TransactionSolidifier txSolidifier; private BlockingQueue preProcessStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue validationStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue receivedStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue replyStageQueue = new ArrayBlockingQueue<>(100); - private BroadcastQueue broadcastStageQueue; + private BlockingQueue broadcastStageQueue = new ArrayBlockingQueue<>(100); /** * Creates a {@link TransactionProcessingPipeline}. @@ -82,9 +88,9 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP * reply stage */ public TransactionProcessingPipelineImpl(NeighborRouter neighborRouter, NodeConfig config, - TransactionValidator txValidator, Tangle tangle, SnapshotProvider snapshotProvider, - TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, - TransactionRequester transactionRequester, BroadcastQueue broadcastStageQueue) { + TransactionValidator txValidator, Tangle tangle, SnapshotProvider snapshotProvider, + TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, + TransactionRequester transactionRequester, TransactionSolidifier txSolidifier) { FIFOCache recentlySeenBytesCache = new FIFOCache<>(config.getCacheSizeBytes()); this.preProcessStage = new PreProcessStage(recentlySeenBytesCache); this.replyStage = new ReplyStage(neighborRouter, config, tangle, tipsViewModel, latestMilestoneTracker, @@ -94,7 +100,7 @@ public TransactionProcessingPipelineImpl(NeighborRouter neighborRouter, NodeConf this.receivedStage = new ReceivedStage(tangle, txValidator, snapshotProvider, transactionRequester); this.batchedHasher = BatchedHasherFactory.create(BatchedHasherFactory.Type.BCTCURL81, 20); this.hashingStage = new HashingStage(batchedHasher); - this.broadcastStageQueue = broadcastStageQueue; + this.txSolidifier = txSolidifier; } @Override @@ -104,7 +110,7 @@ public void start() { addStage("validation", validationStageQueue, validationStage); addStage("reply", replyStageQueue, replyStage); addStage("received", receivedStageQueue, receivedStage); - addStage("broadcast", broadcastStageQueue.get(), broadcastStage); + addStage("broadcast", broadcastStageQueue, broadcastStage); } /** @@ -120,11 +126,8 @@ private void addStage(String name, BlockingQueue queue, try { while (!Thread.currentThread().isInterrupted()) { ProcessingContext queueTake; - if(name.equals("broadcast")) { - queueTake = broadcastStageQueue.get().take(); - } else{ - queueTake = queue.take(); - } + queueTake = queue.take(); + ProcessingContext ctx = stage.process(queueTake); switch (ctx.getNextStage()) { @@ -143,7 +146,7 @@ private void addStage(String name, BlockingQueue queue, receivedStageQueue.put(payload.getRight()); break; case BROADCAST: - broadcastStageQueue.add(ctx); + broadcastStageQueue.put(ctx); break; case ABORT: break; @@ -168,7 +171,7 @@ public BlockingQueue getReceivedStageQueue() { @Override public BlockingQueue getBroadcastStageQueue() { - return broadcastStageQueue.get(); + return broadcastStageQueue; } @Override @@ -185,6 +188,7 @@ public BlockingQueue getValidationStageQueue() { public void process(Neighbor neighbor, ByteBuffer data) { try { preProcessStageQueue.put(new ProcessingContext(new PreProcessPayload(neighbor, data))); + refillBroadcastQueue(); } catch (InterruptedException e) { e.printStackTrace(); } @@ -199,6 +203,23 @@ public void process(byte[] txTrits) { hashAndValidate(new ProcessingContext(payload)); } + @Override + public void refillBroadcastQueue(){ + try{ + Iterator hashIterator = txSolidifier.getBroadcastQueue().iterator(); + Set toRemove = new LinkedHashSet<>(); + while(!Thread.currentThread().isInterrupted() && hashIterator.hasNext()){ + TransactionViewModel t = hashIterator.next(); + broadcastStageQueue.put(new ProcessingContext(new BroadcastPayload(null, t))); + toRemove.add(t); + hashIterator.remove(); + } + txSolidifier.clearBroadcastQueue(toRemove); + } catch(InterruptedException e){ + log.info(e.getMessage()); + } + } + /** * Sets up the given hashing stage {@link ProcessingContext} so that up on success, it will submit further to the * validation stage. diff --git a/src/main/java/com/iota/iri/network/pipeline/ValidationStage.java b/src/main/java/com/iota/iri/network/pipeline/ValidationStage.java index a139210eb4..47c41fa3a9 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ValidationStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ValidationStage.java @@ -1,6 +1,6 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index aa2241a68c..a60fa68740 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -6,7 +6,7 @@ import com.iota.iri.BundleValidator; import com.iota.iri.IRI; import com.iota.iri.IXI; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.APIConfig; import com.iota.iri.conf.IotaConfig; import com.iota.iri.controllers.*; diff --git a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java index 9706f1f9df..feacd6f5e2 100644 --- a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java @@ -1,6 +1,6 @@ package com.iota.iri.service.milestone.impl; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.model.Hash; import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.SnapshotProvider; @@ -44,7 +44,7 @@ public class MilestoneSolidifierImpl implements MilestoneSolidifier { *

*

* Note: We want to find the next previous milestone and not get stuck somewhere at the end of the tangle with a - * long running {@link TransactionValidator#checkSolidity(Hash)} call. + * long running {@link TransactionSolidifier#checkSolidity(Hash)} call. *

*/ private static final int SOLIDIFICATION_TRANSACTIONS_LIMIT = 50000; @@ -60,9 +60,9 @@ public class MilestoneSolidifierImpl implements MilestoneSolidifier { private final SnapshotProvider snapshotProvider; /** - * Holds a reference to the TransactionValidator which allows us to issue solidity checks. + * Holds a reference to the transactionSolidifier which allows us to issue solidity checks. */ - private final TransactionValidator transactionValidator; + private final TransactionSolidifier transactionSolidifier; /** * Holds a reference to the manager of the background worker. @@ -105,11 +105,11 @@ public class MilestoneSolidifierImpl implements MilestoneSolidifier { /** * @param snapshotProvider snapshot provider which gives us access to the relevant snapshots - * @param transactionValidator TransactionValidator instance that is used by the node + * @param transactionSolidifier transactionSolidifier instance that is used by the node */ - public MilestoneSolidifierImpl(SnapshotProvider snapshotProvider, TransactionValidator transactionValidator) { + public MilestoneSolidifierImpl(SnapshotProvider snapshotProvider, TransactionSolidifier transactionSolidifier) { this.snapshotProvider = snapshotProvider; - this.transactionValidator = transactionValidator; + this.transactionSolidifier = transactionSolidifier; } /** @@ -321,7 +321,7 @@ private Map.Entry getNextSolidificationCandidate() { *

*

* It first dumps a log message to keep the node operator informed about the progress of solidification, and then - * issues the {@link TransactionValidator#checkSolidity(Hash, int)} call that starts the solidification + * issues the {@link TransactionSolidifier#checkSolidity(Hash, int)} call that starts the solidification * process. *

*

@@ -341,7 +341,7 @@ private boolean isSolid(Map.Entry currentEntry) { } try { - return transactionValidator.checkSolidity(currentEntry.getKey(), SOLIDIFICATION_TRANSACTIONS_LIMIT); + return transactionSolidifier.checkSolidity(currentEntry.getKey(), SOLIDIFICATION_TRANSACTIONS_LIMIT); } catch (Exception e) { log.error("Error while solidifying milestone #" + currentEntry.getValue(), e); diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java new file mode 100644 index 0000000000..bcf8deb78b --- /dev/null +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -0,0 +1,223 @@ +package com.iota.iri.service.validation; + +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.model.persistables.Milestone; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.snapshot.Snapshot; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.storage.Tangle; +import com.iota.iri.utils.log.interval.IntervalLogger; +import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; +import com.iota.iri.utils.thread.SilentScheduledExecutorService; +import java.util.*; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +import static com.iota.iri.controllers.TransactionViewModel.PREFILLED_SLOT; +import static com.iota.iri.controllers.TransactionViewModel.fromHash; + +public class TransactionSolidifier { + + private Tangle tangle; + private SnapshotProvider snapshotProvider; + private TransactionRequester transactionRequester; + + private static int RESCAN_INTERVAL = 2000; + + private static final IntervalLogger log = new IntervalLogger(TransactionSolidifier.class); + + + private SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( + "Transaction Solidifier", log.delegate()); + + private BlockingQueue transactionsToSolidify = new ArrayBlockingQueue(500); + private BlockingQueue transactionsToUpdate = new ArrayBlockingQueue(500); + + private Set transactionsToBroadcast = new LinkedHashSet<>(); + private Set solidified = new LinkedHashSet<>(); + private Object broadcastSync = new Object(); + + + public TransactionSolidifier(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester){ + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + this.transactionRequester = transactionRequester; + } + + + public void start(){ + executorService.silentScheduleWithFixedDelay(this::transactionSolidifierThread, 0, RESCAN_INTERVAL, + TimeUnit.MILLISECONDS); + } + + + public void shutdown() { + executorService.shutdownNow(); + } + + public boolean addToSolidificationQueue(Hash hash, boolean milestone){ + try{ + if(!solidified.contains(hash) && !transactionsToSolidify.contains(hash)) { + transactionsToSolidify.put(hash); + if(milestone){ + return checkSolidity(hash, 50000); + } + } + + return checkSolidity(hash); + + } catch(Exception e){ + e.getStackTrace(); + return false; + } + } + + + public Set getBroadcastQueue(){ + synchronized (broadcastSync) { + Set broadcastQueue = new LinkedHashSet(); + broadcastQueue.addAll(transactionsToBroadcast); + return broadcastQueue; + } + } + + + public void clearBroadcastQueue(Set transactionsBroadcasted){ + synchronized (broadcastSync) { + for (TransactionViewModel tvm : transactionsBroadcasted) { + transactionsToBroadcast.remove(tvm); + } + } + } + + + private void transactionSolidifierThread(){ + processTransactionsToSolidify(); + processTransactionsToUpdate(); + } + + private void processTransactionsToSolidify(){ + Iterator solidificationIterator = transactionsToSolidify.iterator(); + while(!Thread.currentThread().isInterrupted() && solidificationIterator.hasNext()){ + try { + Hash hash = solidificationIterator.next(); + checkSolidity(hash); + solidificationIterator.remove(); + } catch(Exception e ){ + e.printStackTrace(); + } + } + } + + private void processTransactionsToUpdate(){ + Iterator updateIterator = transactionsToUpdate.iterator(); + while(!Thread.currentThread().isInterrupted() && updateIterator.hasNext()){ + try{ + Hash hash = updateIterator.next(); + updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), hash); + updateIterator.remove(); + } catch(Exception e){ + e.printStackTrace(); + } + } + } + + /** + * This method does the same as {@link #checkSolidity(Hash, int)} but defaults to an unlimited amount + * of transactions that are allowed to be traversed. + * + * @param hash hash of the transactions that shall get checked + * @return true if the transaction is solid and false otherwise + * @throws Exception if anything goes wrong while trying to solidify the transaction + */ + + public boolean checkSolidity(Hash hash) throws Exception { + return checkSolidity(hash, Integer.MAX_VALUE); + } + + /** + * This method checks transactions for solidity and marks them accordingly if they are found to be solid. + * + * It iterates through all approved transactions until it finds one that is missing in the database or until it + * reached solid transactions on all traversed subtangles. In case of a missing transactions it issues a transaction + * request and returns false. If no missing transaction is found, it marks the processed transactions as solid in + * the database and returns true. + * + * Since this operation can potentially take a long time to terminate if it would have to traverse big parts of the + * tangle, it is possible to limit the amount of transactions that are allowed to be processed, while looking for + * unsolid / missing approvees. This can be useful when trying to "interrupt" the solidification of one transaction + * (if it takes too many steps) to give another one the chance to be solidified instead (i.e. prevent blocks in the + * solidification threads). + * + * @param hash hash of the transactions that shall get checked + * @param maxProcessedTransactions the maximum amount of transactions that are allowed to be traversed + * @return true if the transaction is solid and false otherwise + * @throws Exception if anything goes wrong while trying to solidify the transaction + */ + public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception { + if(fromHash(tangle, hash).isSolid()) { + return true; + } + LinkedHashSet analyzedHashes = new LinkedHashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); + if(maxProcessedTransactions != Integer.MAX_VALUE) { + maxProcessedTransactions += analyzedHashes.size(); + } + boolean solid = true; + final Queue nonAnalyzedTransactions = new LinkedList<>(Collections.singleton(hash)); + Hash hashPointer; + while ((hashPointer = nonAnalyzedTransactions.poll()) != null) { + if (!analyzedHashes.add(hashPointer)) { + continue; + } + + if (analyzedHashes.size() >= maxProcessedTransactions) { + return false; + } + + TransactionViewModel transaction = fromHash(tangle, hashPointer); + if (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { + if (transaction.getType() == PREFILLED_SLOT) { + solid = false; + + if (!transactionRequester.isTransactionRequested(hashPointer)) { + transactionRequester.requestTransaction(hashPointer); + continue; + } + } else { + nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); + nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); + } + } + } + if (solid) { + analyzedHashes.forEach(h -> { + try { + solidified.add(h); + transactionsToUpdate.put(h); + } catch(Exception e){ + e.printStackTrace(); + } + }); + } + analyzedHashes.clear(); + return solid; + } + + + public void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, Hash analyzedHash) + throws Exception { + TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, analyzedHash); + + transactionViewModel.updateHeights(tangle, initialSnapshot); + + if (!transactionViewModel.isSolid()) { + transactionViewModel.updateSolid(true); + transactionViewModel.update(tangle, initialSnapshot, "solid|height"); + } + synchronized (broadcastSync) { + transactionsToBroadcast.add(transactionViewModel); + } + } +} diff --git a/src/main/java/com/iota/iri/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java similarity index 79% rename from src/main/java/com/iota/iri/TransactionValidator.java rename to src/main/java/com/iota/iri/service/validation/TransactionValidator.java index a0ee3b01e2..9ffbae1d94 100644 --- a/src/main/java/com/iota/iri/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -1,6 +1,5 @@ -package com.iota.iri; +package com.iota.iri.service.validation; -import com.google.common.annotations.VisibleForTesting; import com.iota.iri.conf.ProtocolConfig; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; @@ -10,7 +9,6 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; -import com.iota.iri.network.pipeline.BroadcastQueue; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; import org.slf4j.Logger; @@ -30,11 +28,11 @@ public class TransactionValidator { private final SnapshotProvider snapshotProvider; private final TipsViewModel tipsViewModel; private final TransactionRequester transactionRequester; + private final TransactionSolidifier transactionSolidifier; private int minWeightMagnitude = 81; private static final long MAX_TIMESTAMP_FUTURE = 2L * 60L * 60L; private static final long MAX_TIMESTAMP_FUTURE_MS = MAX_TIMESTAMP_FUTURE * 1_000L; - private BroadcastQueue broadcastQueue; /////////////////////////////////fields for solidification thread////////////////////////////////////// private Thread newSolidThread; @@ -66,12 +64,13 @@ public class TransactionValidator { * minimum weight magnitude: the minimal number of 9s that ought to appear at the end of the * transaction hash */ - TransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, ProtocolConfig protocolConfig) { + public TransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, ProtocolConfig protocolConfig, TransactionSolidifier transactionSolidifier) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.tipsViewModel = tipsViewModel; this.transactionRequester = transactionRequester; this.newSolidThread = new Thread(spawnSolidTransactionsPropagation(), "Solid TX cascader"); + this.transactionSolidifier = transactionSolidifier; setMwm(protocolConfig.isTestnet(), protocolConfig.getMwm()); } @@ -86,13 +85,11 @@ public class TransactionValidator { * * @see #spawnSolidTransactionsPropagation() */ - public void init(BroadcastQueue broadcastQueue) { - this.broadcastQueue = broadcastQueue; + public void init() { newSolidThread.start(); } - @VisibleForTesting - void setMwm(boolean testnet, int mwm) { + public void setMwm(boolean testnet, int mwm) { minWeightMagnitude = mwm; //lowest allowed MWM encoded in 46 bytes. @@ -212,78 +209,6 @@ public TransactionViewModel validateBytes(final byte[] bytes, int minWeightMagni return transactionViewModel; } - /** - * This method does the same as {@link #checkSolidity(Hash, int)} but defaults to an unlimited amount - * of transactions that are allowed to be traversed. - * - * @param hash hash of the transactions that shall get checked - * @return true if the transaction is solid and false otherwise - * @throws Exception if anything goes wrong while trying to solidify the transaction - */ - public boolean checkSolidity(Hash hash) throws Exception { - return checkSolidity(hash, Integer.MAX_VALUE); - } - - /** - * This method checks transactions for solidity and marks them accordingly if they are found to be solid. - * - * It iterates through all approved transactions until it finds one that is missing in the database or until it - * reached solid transactions on all traversed subtangles. In case of a missing transactions it issues a transaction - * request and returns false. If no missing transaction is found, it marks the processed transactions as solid in - * the database and returns true. - * - * Since this operation can potentially take a long time to terminate if it would have to traverse big parts of the - * tangle, it is possible to limit the amount of transactions that are allowed to be processed, while looking for - * unsolid / missing approvees. This can be useful when trying to "interrupt" the solidification of one transaction - * (if it takes too many steps) to give another one the chance to be solidified instead (i.e. prevent blocks in the - * solidification threads). - * - * @param hash hash of the transactions that shall get checked - * @param maxProcessedTransactions the maximum amount of transactions that are allowed to be traversed - * @return true if the transaction is solid and false otherwise - * @throws Exception if anything goes wrong while trying to solidify the transaction - */ - public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception { - if(fromHash(tangle, hash).isSolid()) { - return true; - } - LinkedHashSet analyzedHashes = new LinkedHashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); - if(maxProcessedTransactions != Integer.MAX_VALUE) { - maxProcessedTransactions += analyzedHashes.size(); - } - boolean solid = true; - final Queue nonAnalyzedTransactions = new LinkedList<>(Collections.singleton(hash)); - Hash hashPointer; - while ((hashPointer = nonAnalyzedTransactions.poll()) != null) { - if (!analyzedHashes.add(hashPointer)) { - continue; - } - - if (analyzedHashes.size() >= maxProcessedTransactions) { - return false; - } - - TransactionViewModel transaction = fromHash(tangle, hashPointer); - if (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { - if (transaction.getType() == PREFILLED_SLOT) { - solid = false; - - if (!transactionRequester.isTransactionRequested(hashPointer)) { - transactionRequester.requestTransaction(hashPointer); - continue; - } - } else { - nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); - nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); - } - } - } - if (solid) { - updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), analyzedHashes, broadcastQueue); - } - analyzedHashes.clear(); - return solid; - } public void addSolidTransaction(Hash hash) { synchronized (cascadeSync) { @@ -318,8 +243,7 @@ private Runnable spawnSolidTransactionsPropagation() { * its children (approvers) and try to quickly solidify them with {@link #quietQuickSetSolid}. * If we manage to solidify the transactions, we add them to the solidification queue for a traversal by a later run. */ - @VisibleForTesting - void propagateSolidTransactions() { + public void propagateSolidTransactions() { Set newSolidHashes = new HashSet<>(); useFirst.set(!useFirst.get()); //synchronized to make sure no one is changing the newSolidTransactions collections during addAll @@ -451,8 +375,7 @@ private boolean checkApproovee(TransactionViewModel approovee) throws Exception return approovee.isSolid(); } - @VisibleForTesting - boolean isNewSolidTxSetsEmpty () { + public boolean isNewSolidTxSetsEmpty () { return newSolidTransactionsOne.isEmpty() && newSolidTransactionsTwo.isEmpty(); } @@ -464,4 +387,5 @@ public static class StaleTimestampException extends RuntimeException { super(message); } } + } diff --git a/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java b/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java index 6e65e21494..911bd9a63b 100644 --- a/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java +++ b/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java @@ -23,6 +23,7 @@ import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPruner; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Tangle; import org.junit.Test; diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java index cdc361d441..c18601d83e 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/TransactionValidatorTest.java @@ -7,9 +7,10 @@ import com.iota.iri.crypto.SpongeFactory; import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; -import com.iota.iri.network.pipeline.BroadcastQueue; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Converter; @@ -44,7 +45,10 @@ public class TransactionValidatorTest { private static SnapshotProvider snapshotProvider; @Mock - private static BroadcastQueue broadcastQueue; + private static TransactionSolidifier txSolidifier; + + @Mock + private static TransactionRequester txRequester; @BeforeClass public static void setUp() throws Exception { @@ -69,16 +73,16 @@ public void setUpEach() { when(snapshotProvider.getInitialSnapshot()).thenReturn(SnapshotMockUtils.createSnapshot()); TipsViewModel tipsViewModel = new TipsViewModel(); TransactionRequester txRequester = new TransactionRequester(tangle, snapshotProvider); - txValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, txRequester, new MainnetConfig()); + txValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, txRequester, new MainnetConfig(), txSolidifier); txValidator.setMwm(false, MAINNET_MWM); - txValidator.init(broadcastQueue); + txValidator.init(); } @Test public void testMinMwm() { ProtocolConfig protocolConfig = mock(ProtocolConfig.class); when(protocolConfig.getMwm()).thenReturn(5); - TransactionValidator transactionValidator = new TransactionValidator(null, null, null, null, protocolConfig); + TransactionValidator transactionValidator = new TransactionValidator(null, null, null, null, protocolConfig, txSolidifier); assertEquals("Expected testnet minimum minWeightMagnitude", 13, transactionValidator.getMinWeightMagnitude()); } @@ -107,15 +111,17 @@ public void validateBytesWithNewCurl() { @Test public void verifyTxIsSolid() throws Exception { TransactionViewModel tx = getTxWithBranchAndTrunk(); - assertTrue(txValidator.checkSolidity(tx.getHash())); - assertTrue(txValidator.checkSolidity(tx.getHash())); + txSolidifier = new TransactionSolidifier(tangle, snapshotProvider, txRequester); + assertTrue(txSolidifier.checkSolidity(tx.getHash())); + assertTrue(txSolidifier.checkSolidity(tx.getHash())); } @Test public void verifyTxIsNotSolid() throws Exception { TransactionViewModel tx = getTxWithoutBranchAndTrunk(); - assertFalse(txValidator.checkSolidity(tx.getHash())); - assertFalse(txValidator.checkSolidity(tx.getHash())); + txSolidifier = new TransactionSolidifier(tangle, snapshotProvider, txRequester); + assertFalse(txSolidifier.checkSolidity(tx.getHash())); + assertFalse(txSolidifier.checkSolidity(tx.getHash())); } @Test diff --git a/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java b/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java index c02749a893..6cb4f26b8e 100644 --- a/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java +++ b/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java @@ -3,7 +3,7 @@ import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.BaseIotaConfig; import com.iota.iri.conf.IotaConfig; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; diff --git a/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java b/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java index 772295c4e5..01e9746782 100644 --- a/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java @@ -1,6 +1,6 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.Neighbor; diff --git a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java index 40d348ae1d..39fa18d255 100644 --- a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java @@ -1,6 +1,7 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.NodeConfig; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.network.NeighborRouter; @@ -18,7 +19,6 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -import java.util.concurrent.BlockingQueue; public class TransactionProcessingPipelineTest { @@ -95,10 +95,7 @@ public class TransactionProcessingPipelineTest { private ProcessingContext abortCtx; @Mock - private BroadcastQueue broadcastQueue; - - @Mock - private BlockingQueue blockingQueue; + private TransactionSolidifier transactionSolidifier; private void mockHashingStage(TransactionProcessingPipeline pipeline) { Mockito.when(hashingPayload.getTxTrits()).thenReturn(null); @@ -122,7 +119,7 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester, broadcastQueue); + transactionRequester, transactionSolidifier); // inject mocks injectMockedStagesIntoPipeline(pipeline); @@ -146,10 +143,6 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); Mockito.when(receivedStage.process(receivedCtx)).thenReturn(broadcastCtx); - // mock broadcastQueue - Mockito.when(broadcastQueue.get()).thenReturn(blockingQueue); - Mockito.when(blockingQueue.take()).thenReturn(broadcastCtx); - pipeline.start(); // send in actual payload to kick off the 'processing' @@ -171,7 +164,7 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws public void processingAKnownTransactionOnlyFlowsToTheReplyStage() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester, broadcastQueue); + transactionRequester, transactionSolidifier); // inject mocks pipeline.setPreProcessStage(preProcessStage); @@ -205,7 +198,7 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester, broadcastQueue); + transactionRequester, transactionSolidifier); // inject mocks injectMockedStagesIntoPipeline(pipeline); @@ -226,10 +219,6 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); Mockito.when(receivedStage.process(receivedCtx)).thenReturn(broadcastCtx); - // mock broadcastQueue - Mockito.when(broadcastQueue.get()).thenReturn(blockingQueue); - Mockito.when(blockingQueue.take()).thenReturn(broadcastCtx); - pipeline.start(); // send into the pipeline but without originating from a neighbor @@ -254,7 +243,7 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug public void anInvalidNewTransactionStopsBeingProcessedAfterTheValidationStage() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester, broadcastQueue); + transactionRequester, transactionSolidifier); // inject mocks injectMockedStagesIntoPipeline(pipeline); diff --git a/src/test/java/com/iota/iri/network/pipeline/ValidationStageTest.java b/src/test/java/com/iota/iri/network/pipeline/ValidationStageTest.java index 5bd0eecdff..c02d57d3b4 100644 --- a/src/test/java/com/iota/iri/network/pipeline/ValidationStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/ValidationStageTest.java @@ -1,6 +1,6 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.network.FIFOCache; diff --git a/src/test/java/com/iota/iri/service/APITest.java b/src/test/java/com/iota/iri/service/APITest.java index f5639156fb..9643959f54 100644 --- a/src/test/java/com/iota/iri/service/APITest.java +++ b/src/test/java/com/iota/iri/service/APITest.java @@ -1,6 +1,6 @@ package com.iota.iri.service; -import com.iota.iri.TransactionValidator; +import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.IotaConfig; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.service.snapshot.SnapshotProvider; From 06242eba19d2c788229e8e2aad15b16b34113d75 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Fri, 8 Nov 2019 01:51:30 -0700 Subject: [PATCH 08/48] Add docs --- .../validation/TransactionSolidifier.java | 124 +++++++++++++++--- 1 file changed, 106 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index bcf8deb78b..5c24e95e18 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -2,7 +2,7 @@ import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.model.persistables.Milestone; +import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.snapshot.SnapshotProvider; @@ -18,63 +18,131 @@ import static com.iota.iri.controllers.TransactionViewModel.PREFILLED_SLOT; import static com.iota.iri.controllers.TransactionViewModel.fromHash; +/** + * A solidifier class for processing transactions. Transactions are checked for solidity, and missing transactions are + * subsequently requested. Once a transaction is solidified correctly it is placed into a broadcasting set to be sent to + * neighboring nodes. + */ public class TransactionSolidifier { private Tangle tangle; private SnapshotProvider snapshotProvider; private TransactionRequester transactionRequester; - private static int RESCAN_INTERVAL = 2000; + /** + * Interval that the {@link #transactionSolidifierThread()} will scan at. + */ + private static int RESCAN_INTERVAL = 5000; private static final IntervalLogger log = new IntervalLogger(TransactionSolidifier.class); - + /** + * Executor service for running the {@link #transactionSolidifierThread()}. + */ private SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( "Transaction Solidifier", log.delegate()); + /** + * A queue for processing transactions with the {@link #checkSolidity(Hash)} call. Once a transaction has been + * marked solid it will be placed into the {@link #transactionsToUpdate} queue. + */ private BlockingQueue transactionsToSolidify = new ArrayBlockingQueue(500); + /** + * A queue for processing transactions with the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} call. + * Once the transaction is updated it is placed into the {@link #transactionsToBroadcast} set. + */ private BlockingQueue transactionsToUpdate = new ArrayBlockingQueue(500); - + /** + * A set of transactions that will be called by the {@link TransactionProcessingPipeline} to be broadcast to + * neighboring nodes. + */ private Set transactionsToBroadcast = new LinkedHashSet<>(); + /** + * A set containing the hash of already solidified transactions + */ private Set solidified = new LinkedHashSet<>(); - private Object broadcastSync = new Object(); + /** + * An object for synchronising access to the {@link #transactionsToBroadcast} set. + */ + private final Object broadcastSync = new Object(); + /** + * Max size and buffer for {@link #solidified} set. + */ + private static int MAX_SIZE= 10000; + private static int BUFFER = 5000; + + /** + * Constructor for the solidifier. + * @param tangle The DB reference + * @param snapshotProvider For fetching entry points for solidity checks + * @param transactionRequester A requester for missing transactions + */ public TransactionSolidifier(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester){ this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.transactionRequester = transactionRequester; } - + /** + * Initialize the executor service. + */ public void start(){ executorService.silentScheduleWithFixedDelay(this::transactionSolidifierThread, 0, RESCAN_INTERVAL, TimeUnit.MILLISECONDS); } - + /** + * Interrupt thread processes and shut down the executor service. + */ public void shutdown() { executorService.shutdownNow(); } - public boolean addToSolidificationQueue(Hash hash, boolean milestone){ + /** + * Add a hash to the solidification queue, and runs an initial {@link #checkSolidity} call. + * + * @param hash Hash of the transaction to solidify + * @return True if the transaction is solid, False if not + */ + public void addToSolidificationQueue(Hash hash){ try{ if(!solidified.contains(hash) && !transactionsToSolidify.contains(hash)) { transactionsToSolidify.put(hash); - if(milestone){ - return checkSolidity(hash, 50000); - } } - return checkSolidity(hash); + checkSolidity(hash); + // Clear room in the solidified set for newer transaction hashes + if(solidified.size() > MAX_SIZE){ + popElderTransactions(); + } + + } catch(Exception e){ + log.error(e.getMessage()); + } + } + /** + * Clears the older half of the {@link #solidified} set to reduce memory footprint. + */ + private void popElderTransactions(){ + try{ + Iterator solidifiedIterator = solidified.iterator(); + for(int i = 0; i < BUFFER; i++){ + solidifiedIterator.next(); + solidifiedIterator.remove(); + } } catch(Exception e){ - e.getStackTrace(); - return false; + log.info(e.getMessage()); } } + /** + * Fetch a copy of the current {@link #transactionsToBroadcast} set. + * @return A set of {@link TransactionViewModel} objects to be broadcast. + */ public Set getBroadcastQueue(){ synchronized (broadcastSync) { Set broadcastQueue = new LinkedHashSet(); @@ -83,7 +151,10 @@ public Set getBroadcastQueue(){ } } - + /** + * Remove any broadcasted transactions from the {@link #transactionsToBroadcast} set + * @param transactionsBroadcasted A set of {@link TransactionViewModel} objects to remove from the set. + */ public void clearBroadcastQueue(Set transactionsBroadcasted){ synchronized (broadcastSync) { for (TransactionViewModel tvm : transactionsBroadcasted) { @@ -92,12 +163,18 @@ public void clearBroadcastQueue(Set transactionsBroadcaste } } - + /** + * Main thread. Process the {@link #transactionsToSolidify} and {@link #transactionsToUpdate} queue's. + */ private void transactionSolidifierThread(){ processTransactionsToSolidify(); processTransactionsToUpdate(); } + /** + * Iterate through the {@link #transactionsToSolidify} queue and call {@link #checkSolidity(Hash)} on each hash. + * Solid transactions are added to the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} set. + */ private void processTransactionsToSolidify(){ Iterator solidificationIterator = transactionsToSolidify.iterator(); while(!Thread.currentThread().isInterrupted() && solidificationIterator.hasNext()){ @@ -111,6 +188,11 @@ private void processTransactionsToSolidify(){ } } + /** + * Iterate through the {@link #transactionsToUpdate} queue and call + * {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} on each hash. Passes updated transactions to the + * {@link #transactionsToBroadcast} set. + */ private void processTransactionsToUpdate(){ Iterator updateIterator = transactionsToUpdate.iterator(); while(!Thread.currentThread().isInterrupted() && updateIterator.hasNext()){ @@ -205,8 +287,14 @@ public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exc return solid; } - - public void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, Hash analyzedHash) + /** + * Updates the solidity of a transaction object and places it into the {@link #transactionsToBroadcast} set. + * @param tangle The DB reference + * @param initialSnapshot Initial snapshot from the {@link #snapshotProvider} + * @param analyzedHash Hash of the transaction that is being updated + * @throws Exception Throws an exception if there is an error updating the transaction object's solidity + */ + private void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, Hash analyzedHash) throws Exception { TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, analyzedHash); From 1ec3794b40550b110a2760d051920733a2e1bc2b Mon Sep 17 00:00:00 2001 From: DyrellC Date: Fri, 8 Nov 2019 01:52:06 -0700 Subject: [PATCH 09/48] Add to solidification queue if update status fails --- .../com/iota/iri/service/validation/TransactionValidator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index 9ffbae1d94..bb5a1eefd3 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -314,6 +314,8 @@ public void updateStatus(TransactionViewModel transactionViewModel) throws Excep transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); tipsViewModel.setSolid(transactionViewModel.getHash()); addSolidTransaction(transactionViewModel.getHash()); + } else { + transactionSolidifier.addToSolidificationQueue(transactionViewModel.getHash()); } } From bc5897f7b6aed99e0477121a7e268590ca5af31a Mon Sep 17 00:00:00 2001 From: DyrellC Date: Fri, 8 Nov 2019 09:49:08 -0700 Subject: [PATCH 10/48] Fix network injection test --- .../iri/network/NetworkInjectionConfigurationTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java b/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java index 6cb4f26b8e..5affbf062e 100644 --- a/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java +++ b/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java @@ -3,6 +3,7 @@ import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; +import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.BaseIotaConfig; import com.iota.iri.conf.IotaConfig; @@ -37,6 +38,11 @@ public void provideTransactionProcessingPipeline() { assertNotNull("instance creation did not work", testInjector().getInstance(TransactionProcessingPipeline.class)); } + @Test + public void provideTransactionSolidifier(){ + assertNotNull("instance creation did not work", testInjector().getInstance(TransactionSolidifier.class)); + } + private Injector testInjector() { IotaConfig config = mock(IotaConfig.class); when(config.getCoordinator()).thenReturn(BaseIotaConfig.Defaults.COORDINATOR); @@ -50,6 +56,7 @@ protected void configure() { bind(LatestMilestoneTracker.class).toInstance(mock(LatestMilestoneTracker.class)); bind(SnapshotProvider.class).toInstance(mock(SnapshotProvider.class)); bind(TransactionValidator.class).toInstance(mock(TransactionValidator.class)); + bind(TransactionSolidifier.class).toInstance(mock(TransactionSolidifier.class)); } } From 62a2330294b7e439b0feb125981ad53424ae29be Mon Sep 17 00:00:00 2001 From: DyrellC Date: Fri, 8 Nov 2019 13:59:28 -0700 Subject: [PATCH 11/48] Remove unused import, lower checkSolidity complexity --- .../TransactionProcessingPipeline.java | 1 - .../validation/TransactionSolidifier.java | 28 +++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java index ec76dae82b..8779286201 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java @@ -1,6 +1,5 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.network.neighbor.Neighbor; import java.nio.ByteBuffer; diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index 5c24e95e18..2ea68b2ff0 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -274,14 +274,7 @@ public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exc } } if (solid) { - analyzedHashes.forEach(h -> { - try { - solidified.add(h); - transactionsToUpdate.put(h); - } catch(Exception e){ - e.printStackTrace(); - } - }); + addToSolidificationQueue(analyzedHashes); } analyzedHashes.clear(); return solid; @@ -296,7 +289,7 @@ public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exc */ private void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, Hash analyzedHash) throws Exception { - TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, analyzedHash); + TransactionViewModel transactionViewModel = fromHash(tangle, analyzedHash); transactionViewModel.updateHeights(tangle, initialSnapshot); @@ -308,4 +301,21 @@ private void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, Ha transactionsToBroadcast.add(transactionViewModel); } } + + /** + * Iterate through analyzed hashes and place them in the {@link #transactionsToUpdate} queue + * @param hashes Analyzed hashes from the {@link #checkSolidity(Hash)} call + */ + private void addToSolidificationQueue(Set hashes){ + hashes.forEach(h -> { + try { + solidified.add(h); + transactionsToUpdate.put(h); + } catch(Exception e){ + e.printStackTrace(); + } + }); + } + + } From 9fc4cbad1337b4376b51c2a3a037cb1e53431670 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Fri, 8 Nov 2019 14:19:48 -0700 Subject: [PATCH 12/48] Reduce complexity further --- .../validation/TransactionSolidifier.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index 2ea68b2ff0..612269f16a 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -259,14 +259,10 @@ public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exc } TransactionViewModel transaction = fromHash(tangle, hashPointer); - if (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { + if (isUnsolidWithoutEntryPoint(transaction, hashPointer)) { if (transaction.getType() == PREFILLED_SLOT) { solid = false; - - if (!transactionRequester.isTransactionRequested(hashPointer)) { - transactionRequester.requestTransaction(hashPointer); - continue; - } + checkRequester(hashPointer); } else { nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); @@ -302,6 +298,16 @@ private void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, Ha } } + /** + * Check if a transaction is present in the {@link #transactionRequester}, if not, it is added. + * @param hashPointer The hash of the transaction to request + */ + private void checkRequester(Hash hashPointer){ + if (!transactionRequester.isTransactionRequested(hashPointer)) { + transactionRequester.requestTransaction(hashPointer); + } + } + /** * Iterate through analyzed hashes and place them in the {@link #transactionsToUpdate} queue * @param hashes Analyzed hashes from the {@link #checkSolidity(Hash)} call @@ -317,5 +323,10 @@ private void addToSolidificationQueue(Set hashes){ }); } - + /** + * Returns true if transaction is not solid and there are no solid entry points from the initial snapshot. + */ + private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Hash hashPointer){ + return (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); + } } From ccca6914c6ac9871d361aba2b356f8fe07799813 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Fri, 8 Nov 2019 14:34:53 -0700 Subject: [PATCH 13/48] Comments --- .../iota/iri/service/validation/TransactionValidator.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index bb5a1eefd3..8d0743cdf2 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -89,6 +89,9 @@ public void init() { newSolidThread.start(); } + /** + * Set the Minimum Weight Magnitude for validation checks. + */ public void setMwm(boolean testnet, int mwm) { minWeightMagnitude = mwm; @@ -377,6 +380,9 @@ private boolean checkApproovee(TransactionViewModel approovee) throws Exception return approovee.isSolid(); } + /** + * Exclusively used in the {@link TansactionValidatorTest} + */ public boolean isNewSolidTxSetsEmpty () { return newSolidTransactionsOne.isEmpty() && newSolidTransactionsTwo.isEmpty(); } From f0c1955a979d520af116ce5084b90105cfeea634 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Tue, 12 Nov 2019 14:35:09 -0700 Subject: [PATCH 14/48] Change queue type, log error messages differentely --- .../validation/TransactionSolidifier.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index 612269f16a..c330bc757d 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -11,8 +11,6 @@ import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; import com.iota.iri.utils.thread.SilentScheduledExecutorService; import java.util.*; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; import static com.iota.iri.controllers.TransactionViewModel.PREFILLED_SLOT; @@ -46,12 +44,12 @@ public class TransactionSolidifier { * A queue for processing transactions with the {@link #checkSolidity(Hash)} call. Once a transaction has been * marked solid it will be placed into the {@link #transactionsToUpdate} queue. */ - private BlockingQueue transactionsToSolidify = new ArrayBlockingQueue(500); + private Deque transactionsToSolidify = new ArrayDeque<>(); /** * A queue for processing transactions with the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} call. * Once the transaction is updated it is placed into the {@link #transactionsToBroadcast} set. */ - private BlockingQueue transactionsToUpdate = new ArrayBlockingQueue(500); + private Deque transactionsToUpdate = new ArrayDeque<>(); /** * A set of transactions that will be called by the {@link TransactionProcessingPipeline} to be broadcast to * neighboring nodes. @@ -109,7 +107,7 @@ public void shutdown() { public void addToSolidificationQueue(Hash hash){ try{ if(!solidified.contains(hash) && !transactionsToSolidify.contains(hash)) { - transactionsToSolidify.put(hash); + transactionsToSolidify.addLast(hash); } checkSolidity(hash); @@ -183,7 +181,7 @@ private void processTransactionsToSolidify(){ checkSolidity(hash); solidificationIterator.remove(); } catch(Exception e ){ - e.printStackTrace(); + log.info(e.getMessage()); } } } @@ -201,7 +199,7 @@ private void processTransactionsToUpdate(){ updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), hash); updateIterator.remove(); } catch(Exception e){ - e.printStackTrace(); + log.info(e.getMessage()); } } } @@ -316,9 +314,9 @@ private void addToSolidificationQueue(Set hashes){ hashes.forEach(h -> { try { solidified.add(h); - transactionsToUpdate.put(h); + transactionsToUpdate.addLast(h); } catch(Exception e){ - e.printStackTrace(); + log.info(e.getMessage()); } }); } From fa72acc0b258b5a3f3617a1617386efca23612a2 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Tue, 12 Nov 2019 14:41:54 -0700 Subject: [PATCH 15/48] Condense code slightly --- .../pipeline/TransactionProcessingPipelineImpl.java | 5 +---- .../iri/service/validation/TransactionSolidifier.java | 10 +++------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index 87cca53582..f1ff3f98dd 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -125,10 +125,7 @@ private void addStage(String name, BlockingQueue queue, stagesThreadPool.submit(new Thread(() -> { try { while (!Thread.currentThread().isInterrupted()) { - ProcessingContext queueTake; - queueTake = queue.take(); - - ProcessingContext ctx = stage.process(queueTake); + ProcessingContext ctx = stage.process(queue.take()); switch (ctx.getNextStage()) { case REPLY: diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index c330bc757d..63ff7ef7ef 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -143,9 +143,7 @@ private void popElderTransactions(){ */ public Set getBroadcastQueue(){ synchronized (broadcastSync) { - Set broadcastQueue = new LinkedHashSet(); - broadcastQueue.addAll(transactionsToBroadcast); - return broadcastQueue; + return new LinkedHashSet(transactionsToBroadcast); } } @@ -177,8 +175,7 @@ private void processTransactionsToSolidify(){ Iterator solidificationIterator = transactionsToSolidify.iterator(); while(!Thread.currentThread().isInterrupted() && solidificationIterator.hasNext()){ try { - Hash hash = solidificationIterator.next(); - checkSolidity(hash); + checkSolidity(solidificationIterator.next()); solidificationIterator.remove(); } catch(Exception e ){ log.info(e.getMessage()); @@ -195,8 +192,7 @@ private void processTransactionsToUpdate(){ Iterator updateIterator = transactionsToUpdate.iterator(); while(!Thread.currentThread().isInterrupted() && updateIterator.hasNext()){ try{ - Hash hash = updateIterator.next(); - updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), hash); + updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), updateIterator.next()); updateIterator.remove(); } catch(Exception e){ log.info(e.getMessage()); From c3b1417debb5bfbf75791d87701db88667056351 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Tue, 12 Nov 2019 21:46:38 -0700 Subject: [PATCH 16/48] Synchronize solidifier queue access, add received transactions to solidifier --- .../TransactionProcessingPipelineImpl.java | 2 + .../validation/TransactionSolidifier.java | 68 ++++++++++++------- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index f1ff3f98dd..d2ca5ad9d3 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -135,6 +135,8 @@ private void addStage(String name, BlockingQueue queue, hashAndValidate(ctx); break; case RECEIVED: + ReceivedPayload receivedPayload = (ReceivedPayload) ctx.getPayload(); + txSolidifier.addToSolidificationQueue(receivedPayload.getTransactionViewModel().getHash()); receivedStageQueue.put(ctx); break; case MULTIPLE: diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index 63ff7ef7ef..3fe8b03513 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -59,6 +59,14 @@ public class TransactionSolidifier { * A set containing the hash of already solidified transactions */ private Set solidified = new LinkedHashSet<>(); + /** + * An object for synchronising access to the {@link #transactionsToSolidify} set. + */ + private final Object solidificationSync = new Object(); + /** + * An object for synchronising access to the {@link #transactionsToUpdate} set. + */ + private final Object updateSync = new Object(); /** * An object for synchronising access to the {@link #transactionsToBroadcast} set. */ @@ -107,7 +115,9 @@ public void shutdown() { public void addToSolidificationQueue(Hash hash){ try{ if(!solidified.contains(hash) && !transactionsToSolidify.contains(hash)) { - transactionsToSolidify.addLast(hash); + synchronized (solidificationSync) { + transactionsToSolidify.add(hash); + } } checkSolidity(hash); @@ -172,13 +182,15 @@ private void transactionSolidifierThread(){ * Solid transactions are added to the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} set. */ private void processTransactionsToSolidify(){ - Iterator solidificationIterator = transactionsToSolidify.iterator(); - while(!Thread.currentThread().isInterrupted() && solidificationIterator.hasNext()){ - try { - checkSolidity(solidificationIterator.next()); - solidificationIterator.remove(); - } catch(Exception e ){ - log.info(e.getMessage()); + synchronized (solidificationSync) { + Iterator solidificationIterator = transactionsToSolidify.iterator(); + while (!Thread.currentThread().isInterrupted() && solidificationIterator.hasNext()) { + try { + checkSolidity(solidificationIterator.next()); + solidificationIterator.remove(); + } catch (Exception e) { + log.info(e.getMessage()); + } } } } @@ -189,13 +201,16 @@ private void processTransactionsToSolidify(){ * {@link #transactionsToBroadcast} set. */ private void processTransactionsToUpdate(){ - Iterator updateIterator = transactionsToUpdate.iterator(); - while(!Thread.currentThread().isInterrupted() && updateIterator.hasNext()){ - try{ - updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), updateIterator.next()); - updateIterator.remove(); - } catch(Exception e){ - log.info(e.getMessage()); + synchronized (updateSync) { + Iterator updateIterator = transactionsToUpdate.iterator(); + while(!Thread.currentThread().isInterrupted() && updateIterator.hasNext()){ + try{ + updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), updateIterator.next()); + updateIterator.remove(); + + } catch(Exception e){ + log.info(e.getMessage()); + } } } } @@ -264,7 +279,7 @@ public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exc } } if (solid) { - addToSolidificationQueue(analyzedHashes); + addToUpdateQueue(analyzedHashes); } analyzedHashes.clear(); return solid; @@ -306,15 +321,18 @@ private void checkRequester(Hash hashPointer){ * Iterate through analyzed hashes and place them in the {@link #transactionsToUpdate} queue * @param hashes Analyzed hashes from the {@link #checkSolidity(Hash)} call */ - private void addToSolidificationQueue(Set hashes){ - hashes.forEach(h -> { - try { - solidified.add(h); - transactionsToUpdate.addLast(h); - } catch(Exception e){ - log.info(e.getMessage()); - } - }); + private void addToUpdateQueue(Set hashes){ + synchronized(updateSync) { + hashes.forEach(h -> { + try { + solidified.add(h); + transactionsToUpdate.add(h); + + } catch (Exception e) { + log.info(e.getMessage()); + } + }); + } } /** From 1538b3633bd189f25b3ef93218a073e951925cd2 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Wed, 13 Nov 2019 23:15:08 -0700 Subject: [PATCH 17/48] Move solidification queue entry point, reduce interval for scans --- .../iota/iri/service/validation/TransactionSolidifier.java | 4 +--- .../com/iota/iri/service/validation/TransactionValidator.java | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index 3fe8b03513..48851716ba 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -30,7 +30,7 @@ public class TransactionSolidifier { /** * Interval that the {@link #transactionSolidifierThread()} will scan at. */ - private static int RESCAN_INTERVAL = 5000; + private static int RESCAN_INTERVAL = 500; private static final IntervalLogger log = new IntervalLogger(TransactionSolidifier.class); @@ -120,12 +120,10 @@ public void addToSolidificationQueue(Hash hash){ } } - checkSolidity(hash); // Clear room in the solidified set for newer transaction hashes if(solidified.size() > MAX_SIZE){ popElderTransactions(); } - } catch(Exception e){ log.error(e.getMessage()); } diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index 8d0743cdf2..267f7bb3bc 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -317,8 +317,6 @@ public void updateStatus(TransactionViewModel transactionViewModel) throws Excep transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); tipsViewModel.setSolid(transactionViewModel.getHash()); addSolidTransaction(transactionViewModel.getHash()); - } else { - transactionSolidifier.addToSolidificationQueue(transactionViewModel.getHash()); } } @@ -329,6 +327,7 @@ public void updateStatus(TransactionViewModel transactionViewModel) throws Excep */ private boolean quietQuickSetSolid(TransactionViewModel transactionViewModel) { try { + transactionSolidifier.addToSolidificationQueue(transactionViewModel.getHash()); return quickSetSolid(transactionViewModel); } catch (Exception e) { log.error(e.getMessage(), e); From a34c5e84994dc8473d9055dcd2c184136058757b Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 18 Nov 2019 07:26:18 -0700 Subject: [PATCH 18/48] Broadcast from reply stage to increase transactions to request speed --- src/main/java/com/iota/iri/network/pipeline/ReplyStage.java | 3 ++- .../network/pipeline/TransactionProcessingPipelineImpl.java | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java b/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java index 9242e978f4..fa94e8456f 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java @@ -142,7 +142,8 @@ public ProcessingContext process(ProcessingContext ctx) { } catch (Exception e) { log.error("error adding reply tx to neighbor's send queue", e); } - ctx.setNextStage(TransactionProcessingPipeline.Stage.ABORT); + ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); + ctx.setPayload(new BroadcastPayload(null, tvm)); return ctx; } diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index d2ca5ad9d3..f1ff3f98dd 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -135,8 +135,6 @@ private void addStage(String name, BlockingQueue queue, hashAndValidate(ctx); break; case RECEIVED: - ReceivedPayload receivedPayload = (ReceivedPayload) ctx.getPayload(); - txSolidifier.addToSolidificationQueue(receivedPayload.getTransactionViewModel().getHash()); receivedStageQueue.put(ctx); break; case MULTIPLE: From 89ec550e0f6536ae62251b5fb728507f2a3d385b Mon Sep 17 00:00:00 2001 From: DyrellC Date: Thu, 21 Nov 2019 08:29:11 -0700 Subject: [PATCH 19/48] Clean up PR, add interface --- .../iota/iri/MainInjectionConfiguration.java | 3 +- .../TransactionProcessingPipeline.java | 3 +- .../TransactionProcessingPipelineImpl.java | 10 +- .../validation/TransactionSolidifier.java | 296 +---------------- .../validation/TransactionValidator.java | 1 + .../impl/TransactionSolidifierImpl.java | 300 ++++++++++++++++++ .../iota/iri/TransactionValidatorTest.java | 5 +- 7 files changed, 326 insertions(+), 292 deletions(-) create mode 100644 src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java diff --git a/src/main/java/com/iota/iri/MainInjectionConfiguration.java b/src/main/java/com/iota/iri/MainInjectionConfiguration.java index 5699addc3c..110524a9dc 100644 --- a/src/main/java/com/iota/iri/MainInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/MainInjectionConfiguration.java @@ -47,6 +47,7 @@ import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; +import com.iota.iri.service.validation.impl.TransactionSolidifierImpl; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.PersistenceProvider; import com.iota.iri.storage.Tangle; @@ -166,7 +167,7 @@ TransactionValidator provideTransactionValidator(Tangle tangle, SnapshotProvider @Singleton @Provides TransactionSolidifier provideTransactionSolidifier(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester){ - return new TransactionSolidifier(tangle, snapshotProvider, transactionRequester); + return new TransactionSolidifierImpl(tangle, snapshotProvider, transactionRequester); } @Singleton diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java index 8779286201..c1c906b493 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java @@ -1,6 +1,7 @@ package com.iota.iri.network.pipeline; import com.iota.iri.network.neighbor.Neighbor; +import com.iota.iri.service.validation.TransactionSolidifier; import java.nio.ByteBuffer; import java.util.concurrent.BlockingQueue; @@ -66,7 +67,7 @@ enum Stage { void process(byte[] txTrits); /** - * Fetches a set of transactions from the {@link com.iota.iri.service.validation.TransactionSolidifier} and submits + * Fetches a set of transactions from the {@link TransactionSolidifier} and submits * the object into the {@link BroadcastStage} queue. */ void refillBroadcastQueue(); diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index f1ff3f98dd..831c4325aa 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -143,6 +143,8 @@ private void addStage(String name, BlockingQueue queue, receivedStageQueue.put(payload.getRight()); break; case BROADCAST: + BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); + txSolidifier.addToSolidificationQueue(broadcastPayload.getTransactionViewModel().getHash()); broadcastStageQueue.put(ctx); break; case ABORT: @@ -206,12 +208,12 @@ public void refillBroadcastQueue(){ Iterator hashIterator = txSolidifier.getBroadcastQueue().iterator(); Set toRemove = new LinkedHashSet<>(); while(!Thread.currentThread().isInterrupted() && hashIterator.hasNext()){ - TransactionViewModel t = hashIterator.next(); - broadcastStageQueue.put(new ProcessingContext(new BroadcastPayload(null, t))); - toRemove.add(t); + TransactionViewModel tx = hashIterator.next(); + broadcastStageQueue.put(new ProcessingContext(new BroadcastPayload(null, tx))); + toRemove.add(tx); hashIterator.remove(); } - txSolidifier.clearBroadcastQueue(toRemove); + txSolidifier.clearFromBroadcastQueue(toRemove); } catch(InterruptedException e){ log.info(e.getMessage()); } diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index 48851716ba..5b4e6f716a 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -2,216 +2,39 @@ import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.network.pipeline.TransactionProcessingPipeline; -import com.iota.iri.network.TransactionRequester; -import com.iota.iri.service.snapshot.Snapshot; -import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.log.interval.IntervalLogger; -import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; -import com.iota.iri.utils.thread.SilentScheduledExecutorService; -import java.util.*; -import java.util.concurrent.TimeUnit; -import static com.iota.iri.controllers.TransactionViewModel.PREFILLED_SLOT; -import static com.iota.iri.controllers.TransactionViewModel.fromHash; +import java.util.Set; -/** - * A solidifier class for processing transactions. Transactions are checked for solidity, and missing transactions are - * subsequently requested. Once a transaction is solidified correctly it is placed into a broadcasting set to be sent to - * neighboring nodes. - */ -public class TransactionSolidifier { - - private Tangle tangle; - private SnapshotProvider snapshotProvider; - private TransactionRequester transactionRequester; - - /** - * Interval that the {@link #transactionSolidifierThread()} will scan at. - */ - private static int RESCAN_INTERVAL = 500; - - private static final IntervalLogger log = new IntervalLogger(TransactionSolidifier.class); +public interface TransactionSolidifier { /** - * Executor service for running the {@link #transactionSolidifierThread()}. + * Initialize the executor service. Start processing transactions to solidify. */ - private SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( - "Transaction Solidifier", log.delegate()); - - /** - * A queue for processing transactions with the {@link #checkSolidity(Hash)} call. Once a transaction has been - * marked solid it will be placed into the {@link #transactionsToUpdate} queue. - */ - private Deque transactionsToSolidify = new ArrayDeque<>(); - /** - * A queue for processing transactions with the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} call. - * Once the transaction is updated it is placed into the {@link #transactionsToBroadcast} set. - */ - private Deque transactionsToUpdate = new ArrayDeque<>(); - /** - * A set of transactions that will be called by the {@link TransactionProcessingPipeline} to be broadcast to - * neighboring nodes. - */ - private Set transactionsToBroadcast = new LinkedHashSet<>(); - /** - * A set containing the hash of already solidified transactions - */ - private Set solidified = new LinkedHashSet<>(); - /** - * An object for synchronising access to the {@link #transactionsToSolidify} set. - */ - private final Object solidificationSync = new Object(); - /** - * An object for synchronising access to the {@link #transactionsToUpdate} set. - */ - private final Object updateSync = new Object(); - /** - * An object for synchronising access to the {@link #transactionsToBroadcast} set. - */ - private final Object broadcastSync = new Object(); - - /** - * Max size and buffer for {@link #solidified} set. - */ - private static int MAX_SIZE= 10000; - private static int BUFFER = 5000; - - - /** - * Constructor for the solidifier. - * @param tangle The DB reference - * @param snapshotProvider For fetching entry points for solidity checks - * @param transactionRequester A requester for missing transactions - */ - public TransactionSolidifier(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester){ - this.tangle = tangle; - this.snapshotProvider = snapshotProvider; - this.transactionRequester = transactionRequester; - } - - /** - * Initialize the executor service. - */ - public void start(){ - executorService.silentScheduleWithFixedDelay(this::transactionSolidifierThread, 0, RESCAN_INTERVAL, - TimeUnit.MILLISECONDS); - } + void start(); /** * Interrupt thread processes and shut down the executor service. */ - public void shutdown() { - executorService.shutdownNow(); - } + void shutdown(); /** * Add a hash to the solidification queue, and runs an initial {@link #checkSolidity} call. * * @param hash Hash of the transaction to solidify - * @return True if the transaction is solid, False if not - */ - public void addToSolidificationQueue(Hash hash){ - try{ - if(!solidified.contains(hash) && !transactionsToSolidify.contains(hash)) { - synchronized (solidificationSync) { - transactionsToSolidify.add(hash); - } - } - - // Clear room in the solidified set for newer transaction hashes - if(solidified.size() > MAX_SIZE){ - popElderTransactions(); - } - } catch(Exception e){ - log.error(e.getMessage()); - } - } - - /** - * Clears the older half of the {@link #solidified} set to reduce memory footprint. */ - private void popElderTransactions(){ - try{ - Iterator solidifiedIterator = solidified.iterator(); - for(int i = 0; i < BUFFER; i++){ - solidifiedIterator.next(); - solidifiedIterator.remove(); - } - } catch(Exception e){ - log.info(e.getMessage()); - } - } - + void addToSolidificationQueue(Hash hash); /** - * Fetch a copy of the current {@link #transactionsToBroadcast} set. + * Fetch a copy of the current transactionsToBroadcast set. * @return A set of {@link TransactionViewModel} objects to be broadcast. */ - public Set getBroadcastQueue(){ - synchronized (broadcastSync) { - return new LinkedHashSet(transactionsToBroadcast); - } - } + Set getBroadcastQueue(); /** - * Remove any broadcasted transactions from the {@link #transactionsToBroadcast} set + * Remove any broadcasted transactions from the transactionsToBroadcast set * @param transactionsBroadcasted A set of {@link TransactionViewModel} objects to remove from the set. */ - public void clearBroadcastQueue(Set transactionsBroadcasted){ - synchronized (broadcastSync) { - for (TransactionViewModel tvm : transactionsBroadcasted) { - transactionsToBroadcast.remove(tvm); - } - } - } - - /** - * Main thread. Process the {@link #transactionsToSolidify} and {@link #transactionsToUpdate} queue's. - */ - private void transactionSolidifierThread(){ - processTransactionsToSolidify(); - processTransactionsToUpdate(); - } - - /** - * Iterate through the {@link #transactionsToSolidify} queue and call {@link #checkSolidity(Hash)} on each hash. - * Solid transactions are added to the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} set. - */ - private void processTransactionsToSolidify(){ - synchronized (solidificationSync) { - Iterator solidificationIterator = transactionsToSolidify.iterator(); - while (!Thread.currentThread().isInterrupted() && solidificationIterator.hasNext()) { - try { - checkSolidity(solidificationIterator.next()); - solidificationIterator.remove(); - } catch (Exception e) { - log.info(e.getMessage()); - } - } - } - } - - /** - * Iterate through the {@link #transactionsToUpdate} queue and call - * {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} on each hash. Passes updated transactions to the - * {@link #transactionsToBroadcast} set. - */ - private void processTransactionsToUpdate(){ - synchronized (updateSync) { - Iterator updateIterator = transactionsToUpdate.iterator(); - while(!Thread.currentThread().isInterrupted() && updateIterator.hasNext()){ - try{ - updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), updateIterator.next()); - updateIterator.remove(); - - } catch(Exception e){ - log.info(e.getMessage()); - } - } - } - } + void clearFromBroadcastQueue(Set transactionsBroadcasted); /** * This method does the same as {@link #checkSolidity(Hash, int)} but defaults to an unlimited amount @@ -221,10 +44,7 @@ private void processTransactionsToUpdate(){ * @return true if the transaction is solid and false otherwise * @throws Exception if anything goes wrong while trying to solidify the transaction */ - - public boolean checkSolidity(Hash hash) throws Exception { - return checkSolidity(hash, Integer.MAX_VALUE); - } + boolean checkSolidity(Hash hash) throws Exception; /** * This method checks transactions for solidity and marks them accordingly if they are found to be solid. @@ -245,98 +65,6 @@ public boolean checkSolidity(Hash hash) throws Exception { * @return true if the transaction is solid and false otherwise * @throws Exception if anything goes wrong while trying to solidify the transaction */ - public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception { - if(fromHash(tangle, hash).isSolid()) { - return true; - } - LinkedHashSet analyzedHashes = new LinkedHashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); - if(maxProcessedTransactions != Integer.MAX_VALUE) { - maxProcessedTransactions += analyzedHashes.size(); - } - boolean solid = true; - final Queue nonAnalyzedTransactions = new LinkedList<>(Collections.singleton(hash)); - Hash hashPointer; - while ((hashPointer = nonAnalyzedTransactions.poll()) != null) { - if (!analyzedHashes.add(hashPointer)) { - continue; - } - - if (analyzedHashes.size() >= maxProcessedTransactions) { - return false; - } + boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception; - TransactionViewModel transaction = fromHash(tangle, hashPointer); - if (isUnsolidWithoutEntryPoint(transaction, hashPointer)) { - if (transaction.getType() == PREFILLED_SLOT) { - solid = false; - checkRequester(hashPointer); - } else { - nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); - nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); - } - } - } - if (solid) { - addToUpdateQueue(analyzedHashes); - } - analyzedHashes.clear(); - return solid; - } - - /** - * Updates the solidity of a transaction object and places it into the {@link #transactionsToBroadcast} set. - * @param tangle The DB reference - * @param initialSnapshot Initial snapshot from the {@link #snapshotProvider} - * @param analyzedHash Hash of the transaction that is being updated - * @throws Exception Throws an exception if there is an error updating the transaction object's solidity - */ - private void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, Hash analyzedHash) - throws Exception { - TransactionViewModel transactionViewModel = fromHash(tangle, analyzedHash); - - transactionViewModel.updateHeights(tangle, initialSnapshot); - - if (!transactionViewModel.isSolid()) { - transactionViewModel.updateSolid(true); - transactionViewModel.update(tangle, initialSnapshot, "solid|height"); - } - synchronized (broadcastSync) { - transactionsToBroadcast.add(transactionViewModel); - } - } - - /** - * Check if a transaction is present in the {@link #transactionRequester}, if not, it is added. - * @param hashPointer The hash of the transaction to request - */ - private void checkRequester(Hash hashPointer){ - if (!transactionRequester.isTransactionRequested(hashPointer)) { - transactionRequester.requestTransaction(hashPointer); - } - } - - /** - * Iterate through analyzed hashes and place them in the {@link #transactionsToUpdate} queue - * @param hashes Analyzed hashes from the {@link #checkSolidity(Hash)} call - */ - private void addToUpdateQueue(Set hashes){ - synchronized(updateSync) { - hashes.forEach(h -> { - try { - solidified.add(h); - transactionsToUpdate.add(h); - - } catch (Exception e) { - log.info(e.getMessage()); - } - }); - } - } - - /** - * Returns true if transaction is not solid and there are no solid entry points from the initial snapshot. - */ - private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Hash hashPointer){ - return (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); - } } diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index 267f7bb3bc..e543aa773f 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -10,6 +10,7 @@ import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.storage.Tangle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java new file mode 100644 index 0000000000..706fa1dbfe --- /dev/null +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -0,0 +1,300 @@ +package com.iota.iri.service.validation.impl; + +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.network.pipeline.TransactionProcessingPipeline; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.snapshot.Snapshot; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.storage.Tangle; +import com.iota.iri.utils.log.interval.IntervalLogger; +import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; +import com.iota.iri.utils.thread.SilentScheduledExecutorService; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import static com.iota.iri.controllers.TransactionViewModel.PREFILLED_SLOT; +import static com.iota.iri.controllers.TransactionViewModel.fromHash; + +/** + * A solidifier class for processing transactions. Transactions are checked for solidity, and missing transactions are + * subsequently requested. Once a transaction is solidified correctly it is placed into a broadcasting set to be sent to + * neighboring nodes. + */ +public class TransactionSolidifierImpl implements TransactionSolidifier { + + private Tangle tangle; + private SnapshotProvider snapshotProvider; + private TransactionRequester transactionRequester; + + /** + * Interval that the {@link #transactionSolidifierThread()} will scan at. + */ + private static final int RESCAN_INTERVAL = 500; + + private static final IntervalLogger log = new IntervalLogger(TransactionSolidifier.class); + + /** + * Executor service for running the {@link #transactionSolidifierThread()}. + */ + private SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( + "Transaction Solidifier", log.delegate()); + + /** + * A queue for processing transactions with the {@link #checkSolidity(Hash)} call. Once a transaction has been + * marked solid it will be placed into the {@link #transactionsToUpdate} queue. + */ + private Queue transactionsToSolidify = new ConcurrentLinkedQueue<>(); + /** + * A queue for processing transactions with the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} call. + * Once the transaction is updated it is placed into the {@link #transactionsToBroadcast} set. + */ + private Queue transactionsToUpdate = new ConcurrentLinkedQueue<>(); + /** + * A set of transactions that will be called by the {@link TransactionProcessingPipeline} to be broadcast to + * neighboring nodes. + */ + private Queue transactionsToBroadcast = new ConcurrentLinkedQueue<>(); + /** + * A set containing the hash of already solidified transactions + */ + private Set solidified = new LinkedHashSet<>(); + + /** + * Max size and buffer for {@link #solidified} set. + */ + private static final int MAX_SIZE= 5000; + private static final int BUFFER = 2500; + + + /** + * Constructor for the solidifier. + * @param tangle The DB reference + * @param snapshotProvider For fetching entry points for solidity checks + * @param transactionRequester A requester for missing transactions + */ + public TransactionSolidifierImpl(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester){ + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + this.transactionRequester = transactionRequester; + } + + /** + *{@inheritDoc} + */ + @Override + public void start(){ + executorService.silentScheduleWithFixedDelay(this::transactionSolidifierThread, 0, RESCAN_INTERVAL, + TimeUnit.MILLISECONDS); + } + + /** + *{@inheritDoc} + */ + @Override + public void shutdown() { + executorService.shutdownNow(); + } + + /** + *{@inheritDoc} + */ + @Override + public void addToSolidificationQueue(Hash hash){ + try{ + if(!solidified.contains(hash) && !transactionsToSolidify.contains(hash)) { + transactionsToSolidify.add(hash); + } + + // Clear room in the solidified set for newer transaction hashes + if(transactionsToSolidify.size() > MAX_SIZE){ + popElderTransactions(); + } + } catch(Exception e){ + log.error(e.getMessage()); + } + } + + /** + * Clears the older half of the {@link #solidified} set to reduce memory footprint. + */ + private void popElderTransactions(){ + try{ + Iterator solidifiedIterator = transactionsToSolidify.iterator(); + for(int i = 0; i < BUFFER; i++){ + solidifiedIterator.next(); + solidifiedIterator.remove(); + } + } catch(Exception e){ + log.info(e.getMessage()); + } + } + + /** + *{@inheritDoc} + */ + @Override + public Set getBroadcastQueue(){ + return new LinkedHashSet<>(transactionsToBroadcast); + } + + /** + *{@inheritDoc} + */ + @Override + public void clearFromBroadcastQueue(Set transactionsBroadcasted){ + for (TransactionViewModel tvm : transactionsBroadcasted) { + transactionsToBroadcast.remove(tvm); + } + } + + /** + * Main thread. Process the {@link #transactionsToSolidify} and {@link #transactionsToUpdate} queue's. + */ + private void transactionSolidifierThread(){ + processTransactionsToSolidify(); + processTransactionsToUpdate(); + + } + + /** + * Iterate through the {@link #transactionsToSolidify} queue and call {@link #checkSolidity(Hash)} on each hash. + * Solid transactions are added to the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} set. + */ + private void processTransactionsToSolidify(){ + Iterator solidificationIterator = transactionsToSolidify.iterator(); + while (!Thread.currentThread().isInterrupted() && solidificationIterator.hasNext()) { + try { + checkSolidity(solidificationIterator.next()); + solidificationIterator.remove(); + } catch (Exception e) { + log.info(e.getMessage()); + } + } + } + + /** + * Iterate through the {@link #transactionsToUpdate} queue and call + * {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} on each hash. Passes updated transactions to the + * {@link #transactionsToBroadcast} set. + */ + private void processTransactionsToUpdate(){ + Iterator updateIterator = transactionsToUpdate.iterator(); + while(!Thread.currentThread().isInterrupted() && updateIterator.hasNext()){ + try{ + updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), updateIterator.next()); + updateIterator.remove(); + + } catch(Exception e){ + log.info(e.getMessage()); + } + } + + } + + /** + *{@inheritDoc} + */ + @Override + public boolean checkSolidity(Hash hash) throws Exception { + return checkSolidity(hash, Integer.MAX_VALUE); + } + + /** + *{@inheritDoc} + */ + @Override + public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception { + if(fromHash(tangle, hash).isSolid()) { + return true; + } + LinkedHashSet analyzedHashes = new LinkedHashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); + if(maxProcessedTransactions != Integer.MAX_VALUE) { + maxProcessedTransactions += analyzedHashes.size(); + } + boolean solid = true; + final Queue nonAnalyzedTransactions = new LinkedList<>(Collections.singleton(hash)); + Hash hashPointer; + while ((hashPointer = nonAnalyzedTransactions.poll()) != null) { + if (!analyzedHashes.add(hashPointer)) { + continue; + } + + if (analyzedHashes.size() >= maxProcessedTransactions) { + return false; + } + + TransactionViewModel transaction = fromHash(tangle, hashPointer); + if (isUnsolidWithoutEntryPoint(transaction, hashPointer)) { + if (transaction.getType() == PREFILLED_SLOT) { + solid = false; + checkRequester(hashPointer); + } else { + nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); + nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); + } + } + } + if (solid) { + addToUpdateQueue(analyzedHashes); + } + analyzedHashes.clear(); + return solid; + } + + /** + * Updates the solidity of a transaction object and places it into the {@link #transactionsToBroadcast} set. + * @param tangle The DB reference + * @param initialSnapshot Initial snapshot from the {@link #snapshotProvider} + * @param analyzedHash Hash of the transaction that is being updated + * @throws Exception Throws an exception if there is an error updating the transaction object's solidity + */ + private void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, Hash analyzedHash) + throws Exception { + TransactionViewModel transactionViewModel = fromHash(tangle, analyzedHash); + + transactionViewModel.updateHeights(tangle, initialSnapshot); + + if (!transactionViewModel.isSolid()) { + transactionViewModel.updateSolid(true); + transactionViewModel.update(tangle, initialSnapshot, "solid|height"); + } + transactionsToBroadcast.add(transactionViewModel); + } + + /** + * Check if a transaction is present in the {@link #transactionRequester}, if not, it is added. + * @param hashPointer The hash of the transaction to request + */ + private void checkRequester(Hash hashPointer){ + if (!transactionRequester.isTransactionRequested(hashPointer)) { + transactionRequester.requestTransaction(hashPointer); + } + } + + /** + * Iterate through analyzed hashes and place them in the {@link #transactionsToUpdate} queue + * @param hashes Analyzed hashes from the {@link #checkSolidity(Hash)} call + */ + private void addToUpdateQueue(Set hashes) { + hashes.forEach(hash -> { + try { + solidified.add(hash); + if(!transactionsToUpdate.contains(hash)) { + transactionsToUpdate.add(hash); + } + } catch (Exception e) { + log.info(e.getMessage()); + } + }); + } + + /** + * Returns true if transaction is not solid and there are no solid entry points from the initial snapshot. + */ + private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Hash hashPointer){ + return (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); + } +} diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java index c18601d83e..9a3d973e4b 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/TransactionValidatorTest.java @@ -11,6 +11,7 @@ import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; +import com.iota.iri.service.validation.impl.TransactionSolidifierImpl; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Converter; @@ -111,7 +112,7 @@ public void validateBytesWithNewCurl() { @Test public void verifyTxIsSolid() throws Exception { TransactionViewModel tx = getTxWithBranchAndTrunk(); - txSolidifier = new TransactionSolidifier(tangle, snapshotProvider, txRequester); + txSolidifier = new TransactionSolidifierImpl(tangle, snapshotProvider, txRequester); assertTrue(txSolidifier.checkSolidity(tx.getHash())); assertTrue(txSolidifier.checkSolidity(tx.getHash())); } @@ -119,7 +120,7 @@ public void verifyTxIsSolid() throws Exception { @Test public void verifyTxIsNotSolid() throws Exception { TransactionViewModel tx = getTxWithoutBranchAndTrunk(); - txSolidifier = new TransactionSolidifier(tangle, snapshotProvider, txRequester); + txSolidifier = new TransactionSolidifierImpl(tangle, snapshotProvider, txRequester); assertFalse(txSolidifier.checkSolidity(tx.getHash())); assertFalse(txSolidifier.checkSolidity(tx.getHash())); } From 8801ef2af790b6ebbac78e164358161658492945 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Thu, 21 Nov 2019 16:53:09 -0700 Subject: [PATCH 20/48] Add unit tests --- .../impl/TransactionSolidifierImpl.java | 6 + .../iota/iri/TransactionValidatorTest.java | 42 ----- .../TransactionProcessingPipelineTest.java | 4 + .../impl/TransactionSolidifierImplTest.java | 144 ++++++++++++++++++ 4 files changed, 154 insertions(+), 42 deletions(-) create mode 100644 src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index 706fa1dbfe..576ef605c1 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -1,5 +1,6 @@ package com.iota.iri.service.validation.impl; +import com.google.common.annotations.VisibleForTesting; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; @@ -297,4 +298,9 @@ private void addToUpdateQueue(Set hashes) { private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Hash hashPointer){ return (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); } + + @VisibleForTesting + Set getSolidificationQueue(){ + return new LinkedHashSet<>(transactionsToSolidify); + } } diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java index 9a3d973e4b..4cf4749ac9 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/TransactionValidatorTest.java @@ -11,7 +11,6 @@ import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; -import com.iota.iri.service.validation.impl.TransactionSolidifierImpl; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Converter; @@ -109,21 +108,6 @@ public void validateBytesWithNewCurl() { txValidator.validateBytes(bytes, txValidator.getMinWeightMagnitude(), SpongeFactory.create(SpongeFactory.Mode.CURLP81)); } - @Test - public void verifyTxIsSolid() throws Exception { - TransactionViewModel tx = getTxWithBranchAndTrunk(); - txSolidifier = new TransactionSolidifierImpl(tangle, snapshotProvider, txRequester); - assertTrue(txSolidifier.checkSolidity(tx.getHash())); - assertTrue(txSolidifier.checkSolidity(tx.getHash())); - } - - @Test - public void verifyTxIsNotSolid() throws Exception { - TransactionViewModel tx = getTxWithoutBranchAndTrunk(); - txSolidifier = new TransactionSolidifierImpl(tangle, snapshotProvider, txRequester); - assertFalse(txSolidifier.checkSolidity(tx.getHash())); - assertFalse(txSolidifier.checkSolidity(tx.getHash())); - } @Test public void addSolidTransactionWithoutErrors() { @@ -132,26 +116,7 @@ public void addSolidTransactionWithoutErrors() { txValidator.addSolidTransaction(TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); } - private TransactionViewModel getTxWithBranchAndTrunk() throws Exception { - TransactionViewModel tx, trunkTx, branchTx; - String trytes = "999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999CFDEZBLZQYA9999999999999999999999999999999999999999999ZZWQHWD99C99999999C99999999CKWWDBWSCLMQULCTAAJGXDEMFJXPMGMAQIHDGHRBGEMUYNNCOK9YPHKEEFLFCZUSPMCJHAKLCIBQSGWAS999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999"; - - byte[] trits = Converter.allocateTritsForTrytes(trytes.length()); - Converter.trits(trytes, trits, 0); - trunkTx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); - branchTx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); - byte[] childTx = getTransactionTrits(); - System.arraycopy(trunkTx.getHash().trits(), 0, childTx, TransactionViewModel.TRUNK_TRANSACTION_TRINARY_OFFSET, TransactionViewModel.TRUNK_TRANSACTION_TRINARY_SIZE); - System.arraycopy(branchTx.getHash().trits(), 0, childTx, TransactionViewModel.BRANCH_TRANSACTION_TRINARY_OFFSET, TransactionViewModel.BRANCH_TRANSACTION_TRINARY_SIZE); - tx = new TransactionViewModel(childTx, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, childTx)); - - trunkTx.store(tangle, snapshotProvider.getInitialSnapshot()); - branchTx.store(tangle, snapshotProvider.getInitialSnapshot()); - tx.store(tangle, snapshotProvider.getInitialSnapshot()); - - return tx; - } @Test public void testTransactionPropagation() throws Exception { @@ -223,12 +188,5 @@ public void testTransactionPropagationFailure() throws Exception { assertFalse("GrandParent tx was expected to be not solid", grandParent.isSolid()); } - private TransactionViewModel getTxWithoutBranchAndTrunk() throws Exception { - byte[] trits = getTransactionTrits(); - TransactionViewModel tx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); - - tx.store(tangle, snapshotProvider.getInitialSnapshot()); - return tx; - } } diff --git a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java index 39fa18d255..39c6914003 100644 --- a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java @@ -76,6 +76,9 @@ public class TransactionProcessingPipelineTest { @Mock private HashingPayload hashingPayload; + @Mock + private BroadcastPayload broadcastPayload; + @Mock private ProcessingContext validationCtx; @@ -141,6 +144,7 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws // mock received Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); + Mockito.when(broadcastCtx.getPayload()).thenReturn(broadcastPayload); Mockito.when(receivedStage.process(receivedCtx)).thenReturn(broadcastCtx); pipeline.start(); diff --git a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java new file mode 100644 index 0000000000..e9ed0a6ed0 --- /dev/null +++ b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java @@ -0,0 +1,144 @@ +package com.iota.iri.service.validation.impl; + +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.crypto.SpongeFactory; +import com.iota.iri.model.TransactionHash; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; +import com.iota.iri.storage.Tangle; +import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; +import com.iota.iri.utils.Converter; +import org.junit.*; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import static com.iota.iri.TransactionTestUtils.getTransactionTrits; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +public class TransactionSolidifierImplTest { + private static final TemporaryFolder dbFolder = new TemporaryFolder(); + private static final TemporaryFolder logFolder = new TemporaryFolder(); + private static Tangle tangle; + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private static SnapshotProvider snapshotProvider; + + @Mock + private static TransactionSolidifierImpl txSolidifier; + + @Mock + private static TransactionRequester txRequester; + + @BeforeClass + public static void setUp() throws Exception { + dbFolder.create(); + logFolder.create(); + tangle = new Tangle(); + tangle.addPersistenceProvider( + new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); + tangle.init(); + } + + @AfterClass + public static void tearDown() throws Exception { + tangle.shutdown(); + dbFolder.delete(); + logFolder.delete(); + } + + @Before + public void setUpEach() { + when(snapshotProvider.getInitialSnapshot()).thenReturn(SnapshotMockUtils.createSnapshot()); + TransactionRequester txRequester = new TransactionRequester(tangle, snapshotProvider); + txSolidifier = new TransactionSolidifierImpl(tangle, snapshotProvider, txRequester); + txSolidifier.start(); + } + + @After + public void tearDownEach(){ + txSolidifier.shutdown(); + } + + + @Test + public void verifyTxIsSolid() throws Exception { + TransactionViewModel tx = getTxWithBranchAndTrunk(); + assertTrue(txSolidifier.checkSolidity(tx.getHash())); + assertTrue(txSolidifier.checkSolidity(tx.getHash())); + } + + @Test + public void verifyTxIsNotSolid() throws Exception { + TransactionViewModel tx = getTxWithoutBranchAndTrunk(); + assertFalse(txSolidifier.checkSolidity(tx.getHash())); + assertFalse(txSolidifier.checkSolidity(tx.getHash())); + } + + @Test + public void getSolidificationQueue() throws Exception{ + TransactionViewModel tx = getTxWithBranchAndTrunk(); + txSolidifier.addToSolidificationQueue(tx.getHash()); + + } + + @Test + public void verifyTransactionIsProcessedFully() throws Exception { + TransactionViewModel tx = getTxWithBranchAndTrunk(); + txSolidifier.addToSolidificationQueue(tx.getHash()); + assertTrue(txSolidifier.getSolidificationQueue().contains(tx.getHash())); + //Time to process through the steps + Thread.sleep(1000); + assertTrue(txSolidifier.getBroadcastQueue().contains(tx)); + } + + + @Test + public void verifyInconsistentTransactionIsNotProcessedFully() throws Exception { + TransactionViewModel tx = getTxWithoutBranchAndTrunk(); + txSolidifier.addToSolidificationQueue(tx.getHash()); + assertTrue(txSolidifier.getSolidificationQueue().contains(tx.getHash())); + //Time to process through the steps + Thread.sleep(1000); + assertFalse(txSolidifier.getBroadcastQueue().contains(tx)); + } + + private TransactionViewModel getTxWithBranchAndTrunk() throws Exception { + TransactionViewModel tx, trunkTx, branchTx; + String trytes = "999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999CFDEZBLZQYA9999999999999999999999999999999999999999999ZZWQHWD99C99999999C99999999CKWWDBWSCLMQULCTAAJGXDEMFJXPMGMAQIHDGHRBGEMUYNNCOK9YPHKEEFLFCZUSPMCJHAKLCIBQSGWAS999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999"; + + byte[] trits = Converter.allocateTritsForTrytes(trytes.length()); + Converter.trits(trytes, trits, 0); + trunkTx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); + branchTx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); + + byte[] childTx = getTransactionTrits(); + System.arraycopy(trunkTx.getHash().trits(), 0, childTx, TransactionViewModel.TRUNK_TRANSACTION_TRINARY_OFFSET, TransactionViewModel.TRUNK_TRANSACTION_TRINARY_SIZE); + System.arraycopy(branchTx.getHash().trits(), 0, childTx, TransactionViewModel.BRANCH_TRANSACTION_TRINARY_OFFSET, TransactionViewModel.BRANCH_TRANSACTION_TRINARY_SIZE); + tx = new TransactionViewModel(childTx, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, childTx)); + + trunkTx.store(tangle, snapshotProvider.getInitialSnapshot()); + branchTx.store(tangle, snapshotProvider.getInitialSnapshot()); + tx.store(tangle, snapshotProvider.getInitialSnapshot()); + + return tx; + } + + private TransactionViewModel getTxWithoutBranchAndTrunk() throws Exception { + byte[] trits = getTransactionTrits(); + TransactionViewModel tx = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); + + tx.store(tangle, snapshotProvider.getInitialSnapshot()); + + return tx; + } + +} From 26b9521df35bb7de58fa4116e6f2a53d0fb68606 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Thu, 21 Nov 2019 16:53:35 -0700 Subject: [PATCH 21/48] Wrap in try/catch, fix styling --- .../TransactionProcessingPipelineImpl.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index 831c4325aa..1498bad9d2 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -88,9 +88,9 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP * reply stage */ public TransactionProcessingPipelineImpl(NeighborRouter neighborRouter, NodeConfig config, - TransactionValidator txValidator, Tangle tangle, SnapshotProvider snapshotProvider, - TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, - TransactionRequester transactionRequester, TransactionSolidifier txSolidifier) { + TransactionValidator txValidator, Tangle tangle, SnapshotProvider snapshotProvider, + TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, + TransactionRequester transactionRequester, TransactionSolidifier txSolidifier) { FIFOCache recentlySeenBytesCache = new FIFOCache<>(config.getCacheSizeBytes()); this.preProcessStage = new PreProcessStage(recentlySeenBytesCache); this.replyStage = new ReplyStage(neighborRouter, config, tangle, tipsViewModel, latestMilestoneTracker, @@ -143,8 +143,12 @@ private void addStage(String name, BlockingQueue queue, receivedStageQueue.put(payload.getRight()); break; case BROADCAST: - BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); - txSolidifier.addToSolidificationQueue(broadcastPayload.getTransactionViewModel().getHash()); + try { + BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); + txSolidifier.addToSolidificationQueue(broadcastPayload.getTransactionViewModel().getHash()); + } catch(Exception e){ + log.info("Error placing transaction in solidifier: " + e.getMessage()); + } broadcastStageQueue.put(ctx); break; case ABORT: From 7e34c917aeaf91613b3cf3824e5e80e63f7925f0 Mon Sep 17 00:00:00 2001 From: Dyrell Chapman Date: Tue, 26 Nov 2019 17:03:34 -0700 Subject: [PATCH 22/48] Update src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java Co-Authored-By: Gal Rogozinski --- .../network/pipeline/TransactionProcessingPipelineImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index 1498bad9d2..79757d0eda 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -145,7 +145,8 @@ private void addStage(String name, BlockingQueue queue, case BROADCAST: try { BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); - txSolidifier.addToSolidificationQueue(broadcastPayload.getTransactionViewModel().getHash()); + TransactionViewModel tvm = broadcastPayload.getTransactionViewModel(); + txSolidifier.addToSolidificationQueue(tvm.getHash()); } catch(Exception e){ log.info("Error placing transaction in solidifier: " + e.getMessage()); } From 778c83325fc06c1c8b85498eabcab0309dc1c3d4 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Tue, 26 Nov 2019 17:29:50 -0700 Subject: [PATCH 23/48] Remove solidification from broadcast queue next stage --- .../network/pipeline/TransactionProcessingPipelineImpl.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index 1498bad9d2..cf35901881 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -143,12 +143,6 @@ private void addStage(String name, BlockingQueue queue, receivedStageQueue.put(payload.getRight()); break; case BROADCAST: - try { - BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); - txSolidifier.addToSolidificationQueue(broadcastPayload.getTransactionViewModel().getHash()); - } catch(Exception e){ - log.info("Error placing transaction in solidifier: " + e.getMessage()); - } broadcastStageQueue.put(ctx); break; case ABORT: From 0b61c896dbc6a199070e926a91a55778b200f594 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Tue, 26 Nov 2019 17:31:05 -0700 Subject: [PATCH 24/48] Add quick solid transactions to broadcast queue, comments and tests update --- .../validation/TransactionSolidifier.java | 12 ++++++++++++ .../validation/TransactionValidator.java | 5 ++++- .../impl/TransactionSolidifierImpl.java | 18 +++++++++++++----- .../com/iota/iri/TransactionValidatorTest.java | 2 +- .../impl/TransactionSolidifierImplTest.java | 2 +- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index 5b4e6f716a..ea47b48638 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -2,9 +2,15 @@ import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +import com.iota.iri.network.TransactionRequester; import java.util.Set; +/** + * Solidification tool. Transactions placed into the solidification queue will be checked for solidity. Any missing + * reference transactions will be placed into the {@link TransactionRequester}. If a transaction is found to be solid + * it is updated as such and placed into the BroadcastQueue to be sent off to the node's neighbours. + */ public interface TransactionSolidifier { /** @@ -67,4 +73,10 @@ public interface TransactionSolidifier { */ boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception; + /** + * Add transaction to the BroadcastQueue set + * @param tvm The transaction to be broadcast + */ + void addToBroadcastQueue(TransactionViewModel tvm); + } diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index e543aa773f..3519a47130 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -1,5 +1,6 @@ package com.iota.iri.service.validation; +import com.google.common.annotations.VisibleForTesting; import com.iota.iri.conf.ProtocolConfig; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; @@ -93,6 +94,7 @@ public void init() { /** * Set the Minimum Weight Magnitude for validation checks. */ + @VisibleForTesting public void setMwm(boolean testnet, int mwm) { minWeightMagnitude = mwm; @@ -328,7 +330,6 @@ public void updateStatus(TransactionViewModel transactionViewModel) throws Excep */ private boolean quietQuickSetSolid(TransactionViewModel transactionViewModel) { try { - transactionSolidifier.addToSolidificationQueue(transactionViewModel.getHash()); return quickSetSolid(transactionViewModel); } catch (Exception e) { log.error(e.getMessage(), e); @@ -355,6 +356,7 @@ private boolean quickSetSolid(final TransactionViewModel transactionViewModel) t if(solid) { transactionViewModel.updateSolid(true); transactionViewModel.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); + transactionSolidifier.addToBroadcastQueue(transactionViewModel); return true; } } @@ -383,6 +385,7 @@ private boolean checkApproovee(TransactionViewModel approovee) throws Exception /** * Exclusively used in the {@link TansactionValidatorTest} */ + @VisibleForTesting public boolean isNewSolidTxSetsEmpty () { return newSolidTransactionsOne.isEmpty() && newSolidTransactionsTwo.isEmpty(); } diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index 576ef605c1..cebcd0998c 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -13,7 +13,7 @@ import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; import com.iota.iri.utils.thread.SilentScheduledExecutorService; import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.TimeUnit; import static com.iota.iri.controllers.TransactionViewModel.PREFILLED_SLOT; @@ -47,17 +47,17 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { * A queue for processing transactions with the {@link #checkSolidity(Hash)} call. Once a transaction has been * marked solid it will be placed into the {@link #transactionsToUpdate} queue. */ - private Queue transactionsToSolidify = new ConcurrentLinkedQueue<>(); + private Queue transactionsToSolidify = new ArrayBlockingQueue<>(1000); /** * A queue for processing transactions with the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} call. * Once the transaction is updated it is placed into the {@link #transactionsToBroadcast} set. */ - private Queue transactionsToUpdate = new ConcurrentLinkedQueue<>(); + private Queue transactionsToUpdate = new ArrayBlockingQueue<>(1000); /** * A set of transactions that will be called by the {@link TransactionProcessingPipeline} to be broadcast to * neighboring nodes. */ - private Queue transactionsToBroadcast = new ConcurrentLinkedQueue<>(); + private Queue transactionsToBroadcast = new ArrayBlockingQueue<>(1000); /** * A set containing the hash of already solidified transactions */ @@ -262,7 +262,7 @@ private void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, Ha transactionViewModel.updateSolid(true); transactionViewModel.update(tangle, initialSnapshot, "solid|height"); } - transactionsToBroadcast.add(transactionViewModel); + addToBroadcastQueue(transactionViewModel); } /** @@ -299,6 +299,14 @@ private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Has return (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); } + /** + * {@inheritDoc} + */ + @Override + public void addToBroadcastQueue(TransactionViewModel tvm){ + transactionsToBroadcast.add(tvm); + } + @VisibleForTesting Set getSolidificationQueue(){ return new LinkedHashSet<>(transactionsToSolidify); diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java index 4cf4749ac9..3d2374793b 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/TransactionValidatorTest.java @@ -72,7 +72,7 @@ public static void tearDown() throws Exception { public void setUpEach() { when(snapshotProvider.getInitialSnapshot()).thenReturn(SnapshotMockUtils.createSnapshot()); TipsViewModel tipsViewModel = new TipsViewModel(); - TransactionRequester txRequester = new TransactionRequester(tangle, snapshotProvider); + txRequester = new TransactionRequester(tangle, snapshotProvider); txValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, txRequester, new MainnetConfig(), txSolidifier); txValidator.setMwm(false, MAINNET_MWM); txValidator.init(); diff --git a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java index e9ed0a6ed0..8991e45184 100644 --- a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java +++ b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java @@ -58,7 +58,7 @@ public static void tearDown() throws Exception { @Before public void setUpEach() { when(snapshotProvider.getInitialSnapshot()).thenReturn(SnapshotMockUtils.createSnapshot()); - TransactionRequester txRequester = new TransactionRequester(tangle, snapshotProvider); + txRequester = new TransactionRequester(tangle, snapshotProvider); txSolidifier = new TransactionSolidifierImpl(tangle, snapshotProvider, txRequester); txSolidifier.start(); } From 864c2dba8d1ac9badadcb03978fe0dc78d81fc89 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Tue, 26 Nov 2019 17:42:15 -0700 Subject: [PATCH 25/48] codacy conflict --- .../validation/impl/TransactionSolidifierImplTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java index 8991e45184..3846f51e7d 100644 --- a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java +++ b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java @@ -9,7 +9,12 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Converter; -import org.junit.*; +import org.junit.Rule; +import org.junit.BeforeClass; +import org.junit.Before; +import org.junit.AfterClass; +import org.junit.After; +import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; From 4bfb1380d4c9f46a3a134f833c8149ba7b247547 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Tue, 26 Nov 2019 23:23:52 -0700 Subject: [PATCH 26/48] Add messages to solidifier test --- .../impl/TransactionSolidifierImplTest.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java index 3846f51e7d..f52718fbd0 100644 --- a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java +++ b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java @@ -77,15 +77,15 @@ public void tearDownEach(){ @Test public void verifyTxIsSolid() throws Exception { TransactionViewModel tx = getTxWithBranchAndTrunk(); - assertTrue(txSolidifier.checkSolidity(tx.getHash())); - assertTrue(txSolidifier.checkSolidity(tx.getHash())); + assertTrue("Expected transaction to be solid", txSolidifier.checkSolidity(tx.getHash())); + assertTrue("Expected transaction to be solid", txSolidifier.checkSolidity(tx.getHash())); } @Test public void verifyTxIsNotSolid() throws Exception { TransactionViewModel tx = getTxWithoutBranchAndTrunk(); - assertFalse(txSolidifier.checkSolidity(tx.getHash())); - assertFalse(txSolidifier.checkSolidity(tx.getHash())); + assertFalse("Expected transaction to fail solidity check", txSolidifier.checkSolidity(tx.getHash())); + assertFalse("Expected transaction to fail solidity check", txSolidifier.checkSolidity(tx.getHash())); } @Test @@ -99,10 +99,12 @@ public void getSolidificationQueue() throws Exception{ public void verifyTransactionIsProcessedFully() throws Exception { TransactionViewModel tx = getTxWithBranchAndTrunk(); txSolidifier.addToSolidificationQueue(tx.getHash()); - assertTrue(txSolidifier.getSolidificationQueue().contains(tx.getHash())); + assertTrue("Expected transaction to be present in the solidification queue", + txSolidifier.getSolidificationQueue().contains(tx.getHash())); //Time to process through the steps Thread.sleep(1000); - assertTrue(txSolidifier.getBroadcastQueue().contains(tx)); + assertTrue("Expected transaction to be present in the broadcast queue", + txSolidifier.getBroadcastQueue().contains(tx)); } @@ -110,10 +112,12 @@ public void verifyTransactionIsProcessedFully() throws Exception { public void verifyInconsistentTransactionIsNotProcessedFully() throws Exception { TransactionViewModel tx = getTxWithoutBranchAndTrunk(); txSolidifier.addToSolidificationQueue(tx.getHash()); - assertTrue(txSolidifier.getSolidificationQueue().contains(tx.getHash())); + assertTrue("Expected transaction to be present in the solidification queue", + txSolidifier.getSolidificationQueue().contains(tx.getHash())); //Time to process through the steps Thread.sleep(1000); - assertFalse(txSolidifier.getBroadcastQueue().contains(tx)); + assertFalse("Expected transaction not to be present in the broadcast queue", + txSolidifier.getBroadcastQueue().contains(tx)); } private TransactionViewModel getTxWithBranchAndTrunk() throws Exception { From d623d1dd073b53ff5b2500f114b3d24d27b778dc Mon Sep 17 00:00:00 2001 From: DyrellC Date: Wed, 27 Nov 2019 01:05:25 -0700 Subject: [PATCH 27/48] Pop eldest on queue fill --- .../impl/TransactionSolidifierImpl.java | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index cebcd0998c..ebad2bacd1 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -34,6 +34,12 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { * Interval that the {@link #transactionSolidifierThread()} will scan at. */ private static final int RESCAN_INTERVAL = 500; + /** + * Max size and buffer for {@link #solidified} set. + */ + private static final int MAX_SIZE= 10000; + private static final int BUFFER= 5000; + private static final IntervalLogger log = new IntervalLogger(TransactionSolidifier.class); @@ -47,27 +53,22 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { * A queue for processing transactions with the {@link #checkSolidity(Hash)} call. Once a transaction has been * marked solid it will be placed into the {@link #transactionsToUpdate} queue. */ - private Queue transactionsToSolidify = new ArrayBlockingQueue<>(1000); + private Queue transactionsToSolidify = new ArrayBlockingQueue<>(MAX_SIZE); /** * A queue for processing transactions with the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} call. * Once the transaction is updated it is placed into the {@link #transactionsToBroadcast} set. */ - private Queue transactionsToUpdate = new ArrayBlockingQueue<>(1000); + private Queue transactionsToUpdate = new ArrayBlockingQueue<>(MAX_SIZE); /** * A set of transactions that will be called by the {@link TransactionProcessingPipeline} to be broadcast to * neighboring nodes. */ - private Queue transactionsToBroadcast = new ArrayBlockingQueue<>(1000); + private Queue transactionsToBroadcast = new ArrayBlockingQueue<>(MAX_SIZE); /** * A set containing the hash of already solidified transactions */ private Set solidified = new LinkedHashSet<>(); - /** - * Max size and buffer for {@link #solidified} set. - */ - private static final int MAX_SIZE= 5000; - private static final int BUFFER = 2500; /** @@ -105,13 +106,12 @@ public void shutdown() { @Override public void addToSolidificationQueue(Hash hash){ try{ - if(!solidified.contains(hash) && !transactionsToSolidify.contains(hash)) { - transactionsToSolidify.add(hash); + if(transactionsToSolidify.size() >= MAX_SIZE){ + transactionsToSolidify.remove(); } - // Clear room in the solidified set for newer transaction hashes - if(transactionsToSolidify.size() > MAX_SIZE){ - popElderTransactions(); + if(!solidified.contains(hash) && !transactionsToSolidify.contains(hash)) { + transactionsToSolidify.add(hash); } } catch(Exception e){ log.error(e.getMessage()); @@ -284,6 +284,10 @@ private void addToUpdateQueue(Set hashes) { try { solidified.add(hash); if(!transactionsToUpdate.contains(hash)) { + if(transactionsToUpdate.size() >= MAX_SIZE){ + transactionsToUpdate.remove(); + } + transactionsToUpdate.add(hash); } } catch (Exception e) { @@ -304,6 +308,9 @@ private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Has */ @Override public void addToBroadcastQueue(TransactionViewModel tvm){ + if(transactionsToBroadcast.size() >= MAX_SIZE){ + transactionsToBroadcast.remove(); + } transactionsToBroadcast.add(tvm); } From 48258b1ea5f8779c887d98c4c586ecfb63fcea0e Mon Sep 17 00:00:00 2001 From: DyrellC Date: Wed, 27 Nov 2019 01:10:13 -0700 Subject: [PATCH 28/48] Remove unused function --- .../impl/TransactionSolidifierImpl.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index ebad2bacd1..263e3b2cbd 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -38,8 +38,6 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { * Max size and buffer for {@link #solidified} set. */ private static final int MAX_SIZE= 10000; - private static final int BUFFER= 5000; - private static final IntervalLogger log = new IntervalLogger(TransactionSolidifier.class); @@ -118,21 +116,6 @@ public void addToSolidificationQueue(Hash hash){ } } - /** - * Clears the older half of the {@link #solidified} set to reduce memory footprint. - */ - private void popElderTransactions(){ - try{ - Iterator solidifiedIterator = transactionsToSolidify.iterator(); - for(int i = 0; i < BUFFER; i++){ - solidifiedIterator.next(); - solidifiedIterator.remove(); - } - } catch(Exception e){ - log.info(e.getMessage()); - } - } - /** *{@inheritDoc} */ From 1cde4bfc147d6831e77b3495283dafb554c20ffc Mon Sep 17 00:00:00 2001 From: DyrellC Date: Wed, 27 Nov 2019 14:53:40 -0700 Subject: [PATCH 29/48] Iterator -> poll() --- .../impl/TransactionSolidifierImpl.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index 263e3b2cbd..00f7e96222 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -148,11 +148,10 @@ private void transactionSolidifierThread(){ * Solid transactions are added to the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} set. */ private void processTransactionsToSolidify(){ - Iterator solidificationIterator = transactionsToSolidify.iterator(); - while (!Thread.currentThread().isInterrupted() && solidificationIterator.hasNext()) { + Hash hash; + while (!Thread.currentThread().isInterrupted() && (hash = transactionsToSolidify.poll())!= null) { try { - checkSolidity(solidificationIterator.next()); - solidificationIterator.remove(); + checkSolidity(hash); } catch (Exception e) { log.info(e.getMessage()); } @@ -165,17 +164,14 @@ private void processTransactionsToSolidify(){ * {@link #transactionsToBroadcast} set. */ private void processTransactionsToUpdate(){ - Iterator updateIterator = transactionsToUpdate.iterator(); - while(!Thread.currentThread().isInterrupted() && updateIterator.hasNext()){ - try{ - updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), updateIterator.next()); - updateIterator.remove(); - - } catch(Exception e){ - log.info(e.getMessage()); - } + Hash hash; + while(!Thread.currentThread().isInterrupted() && (hash = transactionsToUpdate.poll()) != null) { + try { + updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), hash); + } catch (Exception e) { + log.info(e.getMessage()); } - + } } /** From 654398127d4be26697257c71a872c28ec9d28905 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 2 Dec 2019 07:25:06 -0700 Subject: [PATCH 30/48] Add transaction solidification stage --- .../iri/network/pipeline/ReceivedStage.java | 4 +- .../iota/iri/network/pipeline/ReplyStage.java | 3 +- .../iri/network/pipeline/SolidifyPayload.java | 24 +++ .../iri/network/pipeline/SolidifyStage.java | 63 ++++++++ .../TransactionProcessingPipeline.java | 2 +- .../TransactionProcessingPipelineImpl.java | 9 +- .../impl/MilestoneSolidifierImpl.java | 2 +- .../validation/TransactionSolidifier.java | 9 ++ .../validation/TransactionValidator.java | 2 +- .../impl/TransactionSolidifierImpl.java | 142 ++++++++---------- 10 files changed, 169 insertions(+), 91 deletions(-) create mode 100644 src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java create mode 100644 src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java diff --git a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java index 2f07ce7b75..6def7d33e8 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java @@ -91,8 +91,8 @@ public ProcessingContext process(ProcessingContext ctx) { } // broadcast the newly saved tx to the other neighbors - ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); - ctx.setPayload(new BroadcastPayload(originNeighbor, tvm)); + ctx.setNextStage(TransactionProcessingPipeline.Stage.SOLIDIFY); + ctx.setPayload(new SolidifyPayload(originNeighbor, tvm)); return ctx; } } diff --git a/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java b/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java index fa94e8456f..9242e978f4 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java @@ -142,8 +142,7 @@ public ProcessingContext process(ProcessingContext ctx) { } catch (Exception e) { log.error("error adding reply tx to neighbor's send queue", e); } - ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); - ctx.setPayload(new BroadcastPayload(null, tvm)); + ctx.setNextStage(TransactionProcessingPipeline.Stage.ABORT); return ctx; } diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java new file mode 100644 index 0000000000..49314db5cd --- /dev/null +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java @@ -0,0 +1,24 @@ +package com.iota.iri.network.pipeline; + +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.network.neighbor.Neighbor; + +public class SolidifyPayload extends Payload { + Neighbor originNeighbor; + TransactionViewModel tvm; + + public SolidifyPayload(Neighbor originNeighbor, TransactionViewModel tvm){ + this.originNeighbor = originNeighbor; + this.tvm = tvm; + } + + + @Override + public Neighbor getOriginNeighbor(){ + return originNeighbor; + } + + public TransactionViewModel getTransaction(){ + return tvm; + } +} diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java new file mode 100644 index 0000000000..f3f174a877 --- /dev/null +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java @@ -0,0 +1,63 @@ +package com.iota.iri.network.pipeline; + +import com.iota.iri.controllers.TipsViewModel; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; +import com.iota.iri.storage.Tangle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.iota.iri.controllers.TransactionViewModel.fromHash; + +public class SolidifyStage implements Stage { + private static final Logger log = LoggerFactory.getLogger(SolidifyStage.class); + + private TransactionSolidifier txSolidifier; + private TransactionValidator txValidator; + private TipsViewModel tipsViewModel; + private Tangle tangle; + + public SolidifyStage(TransactionSolidifier txSolidifier, TransactionValidator txValidator, + TipsViewModel tipsViewModel, Tangle tangle){ + this.txSolidifier = txSolidifier; + this.txValidator = txValidator; + this.tipsViewModel = tipsViewModel; + this.tangle = tangle; + } + + public ProcessingContext process(ProcessingContext ctx){ + try { + SolidifyPayload payload = (SolidifyPayload) ctx.getPayload(); + TransactionViewModel tvm = payload.getTransaction(); + + if (tvm.isSolid() || txValidator.quickSetSolid(tvm)) { + ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); + ctx.setPayload(new BroadcastPayload(payload.getOriginNeighbor(), payload.getTransaction())); + return ctx; + } + + txSolidifier.addToSolidificationQueue(tvm.getHash()); + + Hash tipHash = tipsViewModel.getRandomSolidTipHash(); + if(tipHash != null) { + TransactionViewModel solidTip = fromHash(tangle, tipHash); + ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); + ctx.setPayload(new BroadcastPayload(payload.getOriginNeighbor(), solidTip)); + return ctx; + } + + ctx.setNextStage(TransactionProcessingPipeline.Stage.FINISH); + return ctx; + }catch (Exception e){ + log.error("Failed to process transaction for solidification", e); + ctx.setNextStage(TransactionProcessingPipeline.Stage.ABORT); + return ctx; + } + + } + + + +} diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java index c1c906b493..59c08c5607 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java @@ -15,7 +15,7 @@ public interface TransactionProcessingPipeline { * Defines the different stages of the {@link TransactionProcessingPipelineImpl}. */ enum Stage { - PRE_PROCESS, HASHING, VALIDATION, REPLY, RECEIVED, BROADCAST, MULTIPLE, ABORT, FINISH, + PRE_PROCESS, HASHING, VALIDATION, REPLY, RECEIVED, BROADCAST, MULTIPLE, ABORT, FINISH, SOLIDIFY, } /** diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index cf35901881..fc0bfe4ff5 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -57,7 +57,7 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingPipeline { private static final Logger log = LoggerFactory.getLogger(TransactionProcessingPipelineImpl.class); - private ExecutorService stagesThreadPool = Executors.newFixedThreadPool(6); + private ExecutorService stagesThreadPool = Executors.newFixedThreadPool(7); // stages of the protocol protocol private PreProcessStage preProcessStage; @@ -67,6 +67,7 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP private BroadcastStage broadcastStage; private BatchedHasher batchedHasher; private HashingStage hashingStage; + private SolidifyStage solidifyStage; private TransactionSolidifier txSolidifier; private BlockingQueue preProcessStageQueue = new ArrayBlockingQueue<>(100); @@ -74,6 +75,7 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP private BlockingQueue receivedStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue replyStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue broadcastStageQueue = new ArrayBlockingQueue<>(100); + private BlockingQueue solidifyStageQueue = new ArrayBlockingQueue<>(100); /** * Creates a {@link TransactionProcessingPipeline}. @@ -100,6 +102,7 @@ public TransactionProcessingPipelineImpl(NeighborRouter neighborRouter, NodeConf this.receivedStage = new ReceivedStage(tangle, txValidator, snapshotProvider, transactionRequester); this.batchedHasher = BatchedHasherFactory.create(BatchedHasherFactory.Type.BCTCURL81, 20); this.hashingStage = new HashingStage(batchedHasher); + this.solidifyStage = new SolidifyStage(txSolidifier, txValidator, tipsViewModel, tangle); this.txSolidifier = txSolidifier; } @@ -111,6 +114,7 @@ public void start() { addStage("reply", replyStageQueue, replyStage); addStage("received", receivedStageQueue, receivedStage); addStage("broadcast", broadcastStageQueue, broadcastStage); + addStage("solidify", solidifyStageQueue, solidifyStage); } /** @@ -145,6 +149,9 @@ private void addStage(String name, BlockingQueue queue, case BROADCAST: broadcastStageQueue.put(ctx); break; + case SOLIDIFY: + solidifyStageQueue.put(ctx); + break; case ABORT: break; case FINISH: diff --git a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java index feacd6f5e2..6a2aa6fc40 100644 --- a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java @@ -341,7 +341,7 @@ private boolean isSolid(Map.Entry currentEntry) { } try { - return transactionSolidifier.checkSolidity(currentEntry.getKey(), SOLIDIFICATION_TRANSACTIONS_LIMIT); + return transactionSolidifier.addMilestoneToSolidificationQueue(currentEntry.getKey(), SOLIDIFICATION_TRANSACTIONS_LIMIT); } catch (Exception e) { log.error("Error while solidifying milestone #" + currentEntry.getValue(), e); diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index ea47b48638..5e75799606 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -30,6 +30,15 @@ public interface TransactionSolidifier { */ void addToSolidificationQueue(Hash hash); + /** + * Checks if milestone transaction is solid. Returns true if it is, and if it is not, it adds the hash to the + * solidification queue and returns false. + * + * @param hash Hash of the transaction to solidify + * @param maxToProcess Maximum number of transactions to analyze + * @return True if solid, false if not + */ + boolean addMilestoneToSolidificationQueue(Hash hash, int maxToProcess); /** * Fetch a copy of the current transactionsToBroadcast set. * @return A set of {@link TransactionViewModel} objects to be broadcast. diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index 3519a47130..3b53e69722 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -344,7 +344,7 @@ private boolean quietQuickSetSolid(TransactionViewModel transactionViewModel) { * @return true if we made the transaction solid, else false. * @throws Exception */ - private boolean quickSetSolid(final TransactionViewModel transactionViewModel) throws Exception { + public boolean quickSetSolid(final TransactionViewModel transactionViewModel) throws Exception { if(!transactionViewModel.isSolid()) { boolean solid = true; if (!checkApproovee(transactionViewModel.getTrunkTransaction(tangle))) { diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index 00f7e96222..0e68942d88 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -5,16 +5,16 @@ import com.iota.iri.model.Hash; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.network.TransactionRequester; -import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.storage.Tangle; import com.iota.iri.utils.log.interval.IntervalLogger; import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; import com.iota.iri.utils.thread.SilentScheduledExecutorService; +import org.apache.commons.collections4.queue.CircularFifoQueue; + import java.util.*; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import static com.iota.iri.controllers.TransactionViewModel.PREFILLED_SLOT; import static com.iota.iri.controllers.TransactionViewModel.fromHash; @@ -30,10 +30,6 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { private SnapshotProvider snapshotProvider; private TransactionRequester transactionRequester; - /** - * Interval that the {@link #transactionSolidifierThread()} will scan at. - */ - private static final int RESCAN_INTERVAL = 500; /** * Max size and buffer for {@link #solidified} set. */ @@ -42,30 +38,29 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { private static final IntervalLogger log = new IntervalLogger(TransactionSolidifier.class); /** - * Executor service for running the {@link #transactionSolidifierThread()}. + * Executor service for running the {@link #processTransactionsToSolidify()}. */ private SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( "Transaction Solidifier", log.delegate()); - /** - * A queue for processing transactions with the {@link #checkSolidity(Hash)} call. Once a transaction has been - * marked solid it will be placed into the {@link #transactionsToUpdate} queue. + * Interval that the {@link #processTransactionsToSolidify()} will scan at. */ - private Queue transactionsToSolidify = new ArrayBlockingQueue<>(MAX_SIZE); + private static final int RESCAN_INTERVAL = 500; /** - * A queue for processing transactions with the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} call. - * Once the transaction is updated it is placed into the {@link #transactionsToBroadcast} set. + * A queue for processing transactions with the {@link #checkSolidity(Hash)} call. Once a transaction has been + * marked solid it will be placed into the {@link #transactionsToBroadcast} queue. */ - private Queue transactionsToUpdate = new ArrayBlockingQueue<>(MAX_SIZE); + private BlockingQueue transactionsToSolidify = new ArrayBlockingQueue<>(MAX_SIZE); + /** * A set of transactions that will be called by the {@link TransactionProcessingPipeline} to be broadcast to * neighboring nodes. */ - private Queue transactionsToBroadcast = new ArrayBlockingQueue<>(MAX_SIZE); + private BlockingQueue transactionsToBroadcast = new ArrayBlockingQueue<>(MAX_SIZE); /** * A set containing the hash of already solidified transactions */ - private Set solidified = new LinkedHashSet<>(); + private Queue solidified = new CircularFifoQueue<>(MAX_SIZE); @@ -86,8 +81,9 @@ public TransactionSolidifierImpl(Tangle tangle, SnapshotProvider snapshotProvide */ @Override public void start(){ - executorService.silentScheduleWithFixedDelay(this::transactionSolidifierThread, 0, RESCAN_INTERVAL, + executorService.silentScheduleWithFixedDelay(this::processTransactionsToSolidify, 0, RESCAN_INTERVAL, TimeUnit.MILLISECONDS); + } /** @@ -104,16 +100,32 @@ public void shutdown() { @Override public void addToSolidificationQueue(Hash hash){ try{ - if(transactionsToSolidify.size() >= MAX_SIZE){ + if(transactionsToSolidify.size() >= MAX_SIZE - 1){ transactionsToSolidify.remove(); } if(!solidified.contains(hash) && !transactionsToSolidify.contains(hash)) { - transactionsToSolidify.add(hash); + transactionsToSolidify.put(hash); } } catch(Exception e){ - log.error(e.getMessage()); + log.error("Error placing transaction into solidification queue",e); + } + } + + @Override + public boolean addMilestoneToSolidificationQueue(Hash hash, int maxToProcess){ + try{ + TransactionViewModel tx = TransactionViewModel.fromHash(tangle, hash); + if(tx.isSolid()){ + return true; + } + addToSolidificationQueue(hash); + return false; + }catch(Exception e){ + log.error("Error adding milestone to solidification queue", e); + return false; } + } /** @@ -134,42 +146,20 @@ public void clearFromBroadcastQueue(Set transactionsBroadc } } - /** - * Main thread. Process the {@link #transactionsToSolidify} and {@link #transactionsToUpdate} queue's. - */ - private void transactionSolidifierThread(){ - processTransactionsToSolidify(); - processTransactionsToUpdate(); - - } /** * Iterate through the {@link #transactionsToSolidify} queue and call {@link #checkSolidity(Hash)} on each hash. - * Solid transactions are added to the {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} set. + * Solid transactions are then processed into the {@link #transactionsToBroadcast} queue. */ private void processTransactionsToSolidify(){ Hash hash; - while (!Thread.currentThread().isInterrupted() && (hash = transactionsToSolidify.poll())!= null) { - try { - checkSolidity(hash); - } catch (Exception e) { - log.info(e.getMessage()); - } - } - } - - /** - * Iterate through the {@link #transactionsToUpdate} queue and call - * {@link #updateSolidTransactions(Tangle, Snapshot, Hash)} on each hash. Passes updated transactions to the - * {@link #transactionsToBroadcast} set. - */ - private void processTransactionsToUpdate(){ - Hash hash; - while(!Thread.currentThread().isInterrupted() && (hash = transactionsToUpdate.poll()) != null) { - try { - updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), hash); - } catch (Exception e) { - log.info(e.getMessage()); + while (!Thread.currentThread().isInterrupted()) { + if((hash = transactionsToSolidify.poll()) != null) { + try { + checkSolidity(hash); + } catch (Exception e) { + log.info(e.getMessage()); + } } } } @@ -218,31 +208,12 @@ public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exc } } if (solid) { - addToUpdateQueue(analyzedHashes); + updateTransactions(analyzedHashes); } analyzedHashes.clear(); return solid; } - /** - * Updates the solidity of a transaction object and places it into the {@link #transactionsToBroadcast} set. - * @param tangle The DB reference - * @param initialSnapshot Initial snapshot from the {@link #snapshotProvider} - * @param analyzedHash Hash of the transaction that is being updated - * @throws Exception Throws an exception if there is an error updating the transaction object's solidity - */ - private void updateSolidTransactions(Tangle tangle, Snapshot initialSnapshot, Hash analyzedHash) - throws Exception { - TransactionViewModel transactionViewModel = fromHash(tangle, analyzedHash); - - transactionViewModel.updateHeights(tangle, initialSnapshot); - - if (!transactionViewModel.isSolid()) { - transactionViewModel.updateSolid(true); - transactionViewModel.update(tangle, initialSnapshot, "solid|height"); - } - addToBroadcastQueue(transactionViewModel); - } /** * Check if a transaction is present in the {@link #transactionRequester}, if not, it is added. @@ -255,20 +226,20 @@ private void checkRequester(Hash hashPointer){ } /** - * Iterate through analyzed hashes and place them in the {@link #transactionsToUpdate} queue + * Iterate through analyzed hashes and place them in the {@link #transactionsToBroadcast} queue * @param hashes Analyzed hashes from the {@link #checkSolidity(Hash)} call */ - private void addToUpdateQueue(Set hashes) { + private void updateTransactions(Set hashes) { hashes.forEach(hash -> { try { - solidified.add(hash); - if(!transactionsToUpdate.contains(hash)) { - if(transactionsToUpdate.size() >= MAX_SIZE){ - transactionsToUpdate.remove(); - } + TransactionViewModel tvm = TransactionViewModel.fromHash(tangle, hash); + tvm.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); - transactionsToUpdate.add(hash); + if(!tvm.isSolid()){ + tvm.updateSolid(true); + tvm.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); } + addToBroadcastQueue(tvm); } catch (Exception e) { log.info(e.getMessage()); } @@ -286,11 +257,16 @@ private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Has * {@inheritDoc} */ @Override - public void addToBroadcastQueue(TransactionViewModel tvm){ - if(transactionsToBroadcast.size() >= MAX_SIZE){ - transactionsToBroadcast.remove(); + public void addToBroadcastQueue(TransactionViewModel tvm) { + try { + if (transactionsToBroadcast.size() >= MAX_SIZE) { + transactionsToBroadcast.remove(); + } + solidified.add(tvm.getHash()); + transactionsToBroadcast.put(tvm); + } catch(Exception e){ + log.error("Error placing transaction into broadcast queue", e); } - transactionsToBroadcast.add(tvm); } @VisibleForTesting From 7b4d3c574355e5516160c0a8b0b410395fad7c5c Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 2 Dec 2019 13:05:11 -0700 Subject: [PATCH 31/48] Comments/clean-up --- .../iri/network/pipeline/ReceivedStage.java | 1 + .../iri/network/pipeline/SolidifyPayload.java | 21 +++++++++++-- .../iri/network/pipeline/SolidifyStage.java | 30 +++++++++++++++++-- .../TransactionProcessingPipeline.java | 7 +++++ .../TransactionProcessingPipelineImpl.java | 5 ++++ .../impl/TransactionSolidifierImpl.java | 4 +-- 6 files changed, 60 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java index 6def7d33e8..f50922bbbc 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java @@ -53,6 +53,7 @@ public ProcessingContext process(ProcessingContext ctx) { boolean stored; try { stored = tvm.store(tangle, snapshotProvider.getInitialSnapshot()); + log.info("STORED!!!!!!!"); } catch (Exception e) { log.error("error persisting newly received tx", e); if (originNeighbor != null) { diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java index 49314db5cd..718a2c88bb 100644 --- a/src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyPayload.java @@ -3,21 +3,36 @@ import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.network.neighbor.Neighbor; +/** + * Defines a payload which gets submitted to the {@link SolidifyStage}. + */ public class SolidifyPayload extends Payload { - Neighbor originNeighbor; - TransactionViewModel tvm; + private Neighbor originNeighbor; + private TransactionViewModel tvm; + /** + * Constructor for solidification payload. + * + * @param originNeighbor The originating point of a received transaction + * @param tvm The transaction that needs to be solidified + */ public SolidifyPayload(Neighbor originNeighbor, TransactionViewModel tvm){ this.originNeighbor = originNeighbor; this.tvm = tvm; } - + /** + * {@inheritDoc} + */ @Override public Neighbor getOriginNeighbor(){ return originNeighbor; } + /** + * Fetches the transaction from the payload. + * @return The transaction stored in the payload. + */ public TransactionViewModel getTransaction(){ return tvm; } diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java index f3f174a877..1dee56838d 100644 --- a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java @@ -11,6 +11,14 @@ import static com.iota.iri.controllers.TransactionViewModel.fromHash; +/** + * The {@link SolidifyStage} is used to process newly received transaction for solidity. Once a transaction has been + * passed from the {@link ReceivedStage} it will be placed into this stage to have the {@link TransactionSolidifier} + * check the solidity of the transaction. If the transaction is found to be solid, it will be passed forward to the + * {@link BroadcastStage}. If it is found to be unsolid, it is put through the solidity check so that missing reference + * transactions get requested. If the transaction is unsolid, a random solid tip is broadcast instead to keep the + * requests transmitting to neighbors. + */ public class SolidifyStage implements Stage { private static final Logger log = LoggerFactory.getLogger(SolidifyStage.class); @@ -19,6 +27,14 @@ public class SolidifyStage implements Stage { private TipsViewModel tipsViewModel; private Tangle tangle; + /** + * Constructor for the {@link SolidifyStage}. + * + * @param txSolidifier Transaction solidifier implementation for updating transaction's solidity status + * @param txValidator Transaction validator implementation for determining the validity of a transaction + * @param tipsViewModel Used for broadcasting random solid tips if the subject transaction is unsolid + * @param tangle A reference to the nodes DB + */ public SolidifyStage(TransactionSolidifier txSolidifier, TransactionValidator txValidator, TipsViewModel tipsViewModel, Tangle tangle){ this.txSolidifier = txSolidifier; @@ -27,6 +43,17 @@ public SolidifyStage(TransactionSolidifier txSolidifier, TransactionValidator tx this.tangle = tangle; } + /** + * Processes the payload of the {@link ProcessingContext} as a {@link SolidifyPayload}. First the transaction will + * be checked for solidity and validity. If the transaction is already solid or can be set solid quickly by the + * transaction validator, the transaction is passed to the {@link BroadcastStage}. If not, the transaction is + * added to the solidification queue, and a random solid tip is pulled form the {@link TipsViewModel} to be + * broadcast instead. + * + * @param ctx The context to process + * @return The output context, in most cases a {@link BroadcastPayload}. + */ + @Override public ProcessingContext process(ProcessingContext ctx){ try { SolidifyPayload payload = (SolidifyPayload) ctx.getPayload(); @@ -57,7 +84,4 @@ public ProcessingContext process(ProcessingContext ctx){ } } - - - } diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java index 59c08c5607..6a659ef7a9 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java @@ -118,4 +118,11 @@ enum Stage { * @param hashingStage the {@link HashingStage} to use */ void setHashingStage(HashingStage hashingStage); + + /** + * Sets the solidify stage. This method should only be used for injecting mocked objects. + * + * @param solidifyStage the {@link SolidifyStage} to use + */ + void setSolidifyStage(SolidifyStage solidifyStage); } diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index fc0bfe4ff5..c401e7beb3 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -281,4 +281,9 @@ public void setBroadcastStage(BroadcastStage broadcastStage) { public void setHashingStage(HashingStage hashingStage) { this.hashingStage = hashingStage; } + + @Override + public void setSolidifyStage(SolidifyStage solidifyStage){ + this.solidifyStage = solidifyStage; + } } diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index 0e68942d88..653808d267 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -115,7 +115,7 @@ public void addToSolidificationQueue(Hash hash){ @Override public boolean addMilestoneToSolidificationQueue(Hash hash, int maxToProcess){ try{ - TransactionViewModel tx = TransactionViewModel.fromHash(tangle, hash); + TransactionViewModel tx = fromHash(tangle, hash); if(tx.isSolid()){ return true; } @@ -232,7 +232,7 @@ private void checkRequester(Hash hashPointer){ private void updateTransactions(Set hashes) { hashes.forEach(hash -> { try { - TransactionViewModel tvm = TransactionViewModel.fromHash(tangle, hash); + TransactionViewModel tvm = fromHash(tangle, hash); tvm.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); if(!tvm.isSolid()){ From 5e0a9743056fa147fbeab8fec09ca4b554042023 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 2 Dec 2019 14:00:58 -0700 Subject: [PATCH 32/48] update tests --- .../network/pipeline/ReceivedStageTest.java | 16 +-- .../network/pipeline/SolidifyStageTest.java | 117 ++++++++++++++++++ .../TransactionProcessingPipelineTest.java | 32 ++++- .../impl/TransactionSolidifierImplTest.java | 15 +-- 4 files changed, 162 insertions(+), 18 deletions(-) create mode 100644 src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java diff --git a/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java b/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java index 01e9746782..db75b68018 100644 --- a/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java @@ -58,11 +58,11 @@ public void newlyStoredTransactionUpdatesAlsoArrivalTimeAndSender() throws Excep Mockito.verify(tvm).update(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(transactionRequester).removeRecentlyRequestedTransaction(Mockito.any()); Mockito.verify(transactionRequester).requestTrunkAndBranch(Mockito.any()); - assertEquals("should submit to broadcast stage next", TransactionProcessingPipeline.Stage.BROADCAST, + assertEquals("should submit to broadcast stage next", TransactionProcessingPipeline.Stage.SOLIDIFY, ctx.getNextStage()); - BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); - assertEquals("neighbor is still the same", neighbor, broadcastPayload.getOriginNeighbor()); - assertEquals("tvm is still the same", tvm, broadcastPayload.getTransactionViewModel()); + SolidifyPayload solidifyPayload = (SolidifyPayload) ctx.getPayload(); + assertEquals("neighbor is still the same", neighbor, solidifyPayload.getOriginNeighbor()); + assertEquals("tvm is still the same", tvm, solidifyPayload.getTransaction()); } @Test @@ -79,11 +79,11 @@ public void alreadyStoredTransactionDoesNoUpdates() throws Exception { Mockito.verify(tvm, Mockito.never()).update(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(transactionRequester).removeRecentlyRequestedTransaction(Mockito.any()); Mockito.verify(transactionRequester, Mockito.never()).requestTrunkAndBranch(Mockito.any()); - assertEquals("should submit to broadcast stage next", TransactionProcessingPipeline.Stage.BROADCAST, + assertEquals("should submit to broadcast stage next", TransactionProcessingPipeline.Stage.SOLIDIFY, ctx.getNextStage()); - BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); - assertEquals("neighbor should still be the same", neighbor, broadcastPayload.getOriginNeighbor()); - assertEquals("tvm should still be the same", tvm, broadcastPayload.getTransactionViewModel()); + SolidifyPayload solidifyPayload = (SolidifyPayload) ctx.getPayload(); + assertEquals("neighbor should still be the same", neighbor, solidifyPayload.getOriginNeighbor()); + assertEquals("tvm should still be the same", tvm, solidifyPayload.getTransaction()); } } \ No newline at end of file diff --git a/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java b/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java new file mode 100644 index 0000000000..17b35f8ab1 --- /dev/null +++ b/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java @@ -0,0 +1,117 @@ +package com.iota.iri.network.pipeline; + +import com.iota.iri.controllers.TipsViewModel; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.service.validation.TransactionValidator; +import com.iota.iri.storage.Tangle; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import static org.junit.Assert.assertEquals; + +public class SolidifyStageTest { + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private Tangle tangle; + + @Mock + private TipsViewModel tipsViewModel; + + @Mock + private TransactionViewModel tvm; + + @Mock + private Hash originalHash; + + @Mock + private Hash tipHash; + + @Mock + private TransactionSolidifier transactionSolidifier; + + @Mock + private TransactionValidator transactionValidator; + + @Test + public void solidTransactionIsBroadcast() throws Exception{ + Mockito.when(tvm.isSolid()).thenReturn(true); + Mockito.when(tvm.getHash()).thenReturn(originalHash); + + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, transactionValidator, tipsViewModel, tangle); + SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); + ProcessingContext ctx = new ProcessingContext(solidifyPayload); + + solidifyStage.process(ctx); + Thread.sleep(100); + + assertEquals("Expected next stage to be broadcast", ctx.getNextStage(), + TransactionProcessingPipeline.Stage.BROADCAST); + BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); + assertEquals("Expected payload hash to equal the original transaction hash", + broadcastPayload.getTransactionViewModel().getHash(), originalHash); + } + + @Test + public void quickSetSolidTransactionIsBroadcast() throws Exception{ + Mockito.when(transactionValidator.quickSetSolid(tvm)).thenReturn(true); + Mockito.when(tvm.getHash()).thenReturn(originalHash); + + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, transactionValidator, tipsViewModel, tangle); + SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); + ProcessingContext ctx = new ProcessingContext(solidifyPayload); + + solidifyStage.process(ctx); + Thread.sleep(100); + + assertEquals("Expected next stage to be broadcast", ctx.getNextStage(), + TransactionProcessingPipeline.Stage.BROADCAST); + BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); + assertEquals("Expected payload hash to equal the original transaction hash", + broadcastPayload.getTransactionViewModel().getHash(), originalHash); + } + + @Test + public void unsolidTransactionBroadcastsRandomSolidTip() throws Exception{ + Mockito.when(tvm.isSolid()).thenReturn(false); + Mockito.when(transactionValidator.quickSetSolid(tvm)).thenReturn(false); + Mockito.when(tipsViewModel.getRandomSolidTipHash()).thenReturn(tipHash); + + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, transactionValidator, tipsViewModel, tangle); + SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); + ProcessingContext ctx = new ProcessingContext(solidifyPayload); + + solidifyStage.process(ctx); + Thread.sleep(100); + + assertEquals("Expected next stage to be broadcast", ctx.getNextStage(), + TransactionProcessingPipeline.Stage.BROADCAST); + BroadcastPayload broadcastPayload = (BroadcastPayload) ctx.getPayload(); + assertEquals("Expected payload hash to equal random tip hash", + broadcastPayload.getTransactionViewModel().getHash(), tipHash); + } + + @Test + public void unsolidWithNoRandomTipsAborts() throws Exception{ + Mockito.when(tvm.isSolid()).thenReturn(false); + Mockito.when(transactionValidator.quickSetSolid(tvm)).thenReturn(false); + Mockito.when(tipsViewModel.getRandomSolidTipHash()).thenReturn(null); + + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, transactionValidator, tipsViewModel, tangle); + SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); + ProcessingContext ctx = new ProcessingContext(solidifyPayload); + + solidifyStage.process(ctx); + Thread.sleep(100); + + assertEquals("Expected next stage to be broadcast", ctx.getNextStage(), + TransactionProcessingPipeline.Stage.FINISH); + } +} diff --git a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java index 39c6914003..97d323ee14 100644 --- a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java @@ -79,6 +79,15 @@ public class TransactionProcessingPipelineTest { @Mock private BroadcastPayload broadcastPayload; + @Mock + private ReceivedPayload receivedPayload; + + @Mock + private SolidifyStage solidifyStage; + + @Mock + private SolidifyPayload solidifyPayload; + @Mock private ProcessingContext validationCtx; @@ -94,6 +103,9 @@ public class TransactionProcessingPipelineTest { @Mock private ProcessingContext broadcastCtx; + @Mock + private ProcessingContext solidifyCtx; + @Mock private ProcessingContext abortCtx; @@ -115,6 +127,7 @@ private void injectMockedStagesIntoPipeline(TransactionProcessingPipeline pipeli pipeline.setHashingStage(hashingStage); pipeline.setReplyStage(replyStage); pipeline.setValidationStage(validationStage); + pipeline.setSolidifyStage(solidifyStage); } @Test @@ -145,7 +158,12 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws // mock received Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); Mockito.when(broadcastCtx.getPayload()).thenReturn(broadcastPayload); - Mockito.when(receivedStage.process(receivedCtx)).thenReturn(broadcastCtx); + Mockito.when(receivedStage.process(receivedCtx)).thenReturn(solidifyCtx); + + // mock solidify + Mockito.when(solidifyCtx.getPayload()).thenReturn(solidifyPayload); + Mockito.when(solidifyCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.SOLIDIFY); + Mockito.when(solidifyStage.process(solidifyCtx)).thenReturn(broadcastCtx); pipeline.start(); @@ -161,6 +179,7 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws Mockito.verify(validationStage).process(Mockito.any()); Mockito.verify(receivedStage).process(Mockito.any()); Mockito.verify(replyStage).process(Mockito.any()); + Mockito.verify(solidifyStage).process(Mockito.any()); Mockito.verify(broadcastStage).process(Mockito.any()); } @@ -190,6 +209,7 @@ public void processingAKnownTransactionOnlyFlowsToTheReplyStage() throws Interru Mockito.verify(hashingStage, Mockito.never()).process(Mockito.any()); Mockito.verify(validationStage, Mockito.never()).process(Mockito.any()); Mockito.verify(receivedStage, Mockito.never()).process(Mockito.any()); + Mockito.verify(solidifyStage, Mockito.never()).process(Mockito.any()); Mockito.verify(broadcastStage, Mockito.never()).process(Mockito.any()); // should have called @@ -211,7 +231,6 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug Mockito.when(hashingCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.HASHING); Mockito.when(hashingCtx.getPayload()).thenReturn(hashingPayload); - // mock hashing context/stage // mock hashing context/stage mockHashingStage(pipeline); @@ -221,7 +240,12 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug // mock received Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); - Mockito.when(receivedStage.process(receivedCtx)).thenReturn(broadcastCtx); + Mockito.when(receivedStage.process(receivedCtx)).thenReturn(solidifyCtx); + + // mock solidify + Mockito.when(solidifyCtx.getPayload()).thenReturn(solidifyPayload); + Mockito.when(solidifyCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.SOLIDIFY); + Mockito.when(solidifyStage.process(solidifyCtx)).thenReturn(broadcastCtx); pipeline.start(); @@ -240,6 +264,7 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug Mockito.verify(hashingStage).process(Mockito.any()); Mockito.verify(validationStage).process(Mockito.any()); Mockito.verify(receivedStage).process(Mockito.any()); + Mockito.verify(solidifyStage).process(Mockito.any()); Mockito.verify(broadcastStage).process(Mockito.any()); } @@ -278,6 +303,7 @@ public void anInvalidNewTransactionStopsBeingProcessedAfterTheValidationStage() Mockito.verify(preProcessStage, Mockito.never()).process(Mockito.any()); Mockito.verify(broadcastStage, Mockito.never()).process(Mockito.any()); Mockito.verify(receivedStage, Mockito.never()).process(Mockito.any()); + Mockito.verify(solidifyStage, Mockito.never()).process(Mockito.any()); // should have called Mockito.verify(hashingStage).process(Mockito.any()); diff --git a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java index f52718fbd0..3ad4bf8d63 100644 --- a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java +++ b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java @@ -89,18 +89,20 @@ public void verifyTxIsNotSolid() throws Exception { } @Test - public void getSolidificationQueue() throws Exception{ + public void getSolidificationQueue() throws Exception { TransactionViewModel tx = getTxWithBranchAndTrunk(); - txSolidifier.addToSolidificationQueue(tx.getHash()); - + for(int i = 0; i < 10; i++) { + txSolidifier.addToSolidificationQueue(tx.getHash()); + } + assertTrue("Expected transaction to be present in the solidification queue", + txSolidifier.getSolidificationQueue().contains(tx.getHash())); } @Test public void verifyTransactionIsProcessedFully() throws Exception { TransactionViewModel tx = getTxWithBranchAndTrunk(); txSolidifier.addToSolidificationQueue(tx.getHash()); - assertTrue("Expected transaction to be present in the solidification queue", - txSolidifier.getSolidificationQueue().contains(tx.getHash())); + //Time to process through the steps Thread.sleep(1000); assertTrue("Expected transaction to be present in the broadcast queue", @@ -112,8 +114,7 @@ public void verifyTransactionIsProcessedFully() throws Exception { public void verifyInconsistentTransactionIsNotProcessedFully() throws Exception { TransactionViewModel tx = getTxWithoutBranchAndTrunk(); txSolidifier.addToSolidificationQueue(tx.getHash()); - assertTrue("Expected transaction to be present in the solidification queue", - txSolidifier.getSolidificationQueue().contains(tx.getHash())); + //Time to process through the steps Thread.sleep(1000); assertFalse("Expected transaction not to be present in the broadcast queue", From f86fe55b0ed60587b766f5720fbc3d6357baab4a Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 2 Dec 2019 14:09:17 -0700 Subject: [PATCH 33/48] Remove unused payload Mock --- .../network/pipeline/TransactionProcessingPipelineTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java index 97d323ee14..576f418bc0 100644 --- a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java @@ -79,9 +79,6 @@ public class TransactionProcessingPipelineTest { @Mock private BroadcastPayload broadcastPayload; - @Mock - private ReceivedPayload receivedPayload; - @Mock private SolidifyStage solidifyStage; From de08907cb799fd1db7d282ef0ddf578ad4fe55ee Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 2 Dec 2019 15:28:11 -0700 Subject: [PATCH 34/48] Remove log --- src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java index f50922bbbc..6def7d33e8 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java @@ -53,7 +53,6 @@ public ProcessingContext process(ProcessingContext ctx) { boolean stored; try { stored = tvm.store(tangle, snapshotProvider.getInitialSnapshot()); - log.info("STORED!!!!!!!"); } catch (Exception e) { log.error("error persisting newly received tx", e); if (originNeighbor != null) { From 4086fce4e64f381b133525ea7b866b82ecec80c9 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 2 Dec 2019 16:05:22 -0700 Subject: [PATCH 35/48] Remove circular solidified queue --- .../validation/impl/TransactionSolidifierImpl.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index 653808d267..02458cb105 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -11,7 +11,6 @@ import com.iota.iri.utils.log.interval.IntervalLogger; import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; import com.iota.iri.utils.thread.SilentScheduledExecutorService; -import org.apache.commons.collections4.queue.CircularFifoQueue; import java.util.*; import java.util.concurrent.*; @@ -31,7 +30,7 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { private TransactionRequester transactionRequester; /** - * Max size and buffer for {@link #solidified} set. + * Max size for all queues. */ private static final int MAX_SIZE= 10000; @@ -57,11 +56,6 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { * neighboring nodes. */ private BlockingQueue transactionsToBroadcast = new ArrayBlockingQueue<>(MAX_SIZE); - /** - * A set containing the hash of already solidified transactions - */ - private Queue solidified = new CircularFifoQueue<>(MAX_SIZE); - /** @@ -104,7 +98,7 @@ public void addToSolidificationQueue(Hash hash){ transactionsToSolidify.remove(); } - if(!solidified.contains(hash) && !transactionsToSolidify.contains(hash)) { + if(!transactionsToSolidify.contains(hash)) { transactionsToSolidify.put(hash); } } catch(Exception e){ @@ -262,7 +256,7 @@ public void addToBroadcastQueue(TransactionViewModel tvm) { if (transactionsToBroadcast.size() >= MAX_SIZE) { transactionsToBroadcast.remove(); } - solidified.add(tvm.getHash()); + transactionsToBroadcast.put(tvm); } catch(Exception e){ log.error("Error placing transaction into broadcast queue", e); From b16c752b05b14916410cee59fb59b4931b32b92f Mon Sep 17 00:00:00 2001 From: DyrellC Date: Wed, 4 Dec 2019 11:22:45 -0700 Subject: [PATCH 36/48] Make addToBroadcastQueue private again --- .../java/com/iota/iri/MainInjectionConfiguration.java | 4 ++-- .../iri/service/validation/TransactionSolidifier.java | 7 ------- .../iota/iri/service/validation/TransactionValidator.java | 6 +----- .../validation/impl/TransactionSolidifierImpl.java | 7 ++----- src/test/java/com/iota/iri/TransactionValidatorTest.java | 8 ++------ 5 files changed, 7 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/iota/iri/MainInjectionConfiguration.java b/src/main/java/com/iota/iri/MainInjectionConfiguration.java index ecc16f9d6d..c22e4879f5 100644 --- a/src/main/java/com/iota/iri/MainInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/MainInjectionConfiguration.java @@ -162,8 +162,8 @@ LocalSnapshotManager provideLocalSnapshotManager(SnapshotProvider snapshotProvid @Singleton @Provides - TransactionValidator provideTransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, TransactionSolidifier transactionSolidifier) { - return new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester, configuration, transactionSolidifier); + TransactionValidator provideTransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester) { + return new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester, configuration); } @Singleton diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index 5e75799606..9899ec4f9d 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -81,11 +81,4 @@ public interface TransactionSolidifier { * @throws Exception if anything goes wrong while trying to solidify the transaction */ boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception; - - /** - * Add transaction to the BroadcastQueue set - * @param tvm The transaction to be broadcast - */ - void addToBroadcastQueue(TransactionViewModel tvm); - } diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index 3b53e69722..d190baa183 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -11,7 +11,6 @@ import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.storage.Tangle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +29,6 @@ public class TransactionValidator { private final SnapshotProvider snapshotProvider; private final TipsViewModel tipsViewModel; private final TransactionRequester transactionRequester; - private final TransactionSolidifier transactionSolidifier; private int minWeightMagnitude = 81; private static final long MAX_TIMESTAMP_FUTURE = 2L * 60L * 60L; private static final long MAX_TIMESTAMP_FUTURE_MS = MAX_TIMESTAMP_FUTURE * 1_000L; @@ -66,13 +64,12 @@ public class TransactionValidator { * minimum weight magnitude: the minimal number of 9s that ought to appear at the end of the * transaction hash */ - public TransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, ProtocolConfig protocolConfig, TransactionSolidifier transactionSolidifier) { + public TransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, ProtocolConfig protocolConfig) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.tipsViewModel = tipsViewModel; this.transactionRequester = transactionRequester; this.newSolidThread = new Thread(spawnSolidTransactionsPropagation(), "Solid TX cascader"); - this.transactionSolidifier = transactionSolidifier; setMwm(protocolConfig.isTestnet(), protocolConfig.getMwm()); } @@ -356,7 +353,6 @@ public boolean quickSetSolid(final TransactionViewModel transactionViewModel) th if(solid) { transactionViewModel.updateSolid(true); transactionViewModel.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); - transactionSolidifier.addToBroadcastQueue(transactionViewModel); return true; } } diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index 02458cb105..1052be14d6 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -247,11 +247,8 @@ private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Has return (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); } - /** - * {@inheritDoc} - */ - @Override - public void addToBroadcastQueue(TransactionViewModel tvm) { + + private void addToBroadcastQueue(TransactionViewModel tvm) { try { if (transactionsToBroadcast.size() >= MAX_SIZE) { transactionsToBroadcast.remove(); diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java index 3d2374793b..7314a3d8cc 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/TransactionValidatorTest.java @@ -9,7 +9,6 @@ import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; -import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; @@ -44,9 +43,6 @@ public class TransactionValidatorTest { @Mock private static SnapshotProvider snapshotProvider; - @Mock - private static TransactionSolidifier txSolidifier; - @Mock private static TransactionRequester txRequester; @@ -73,7 +69,7 @@ public void setUpEach() { when(snapshotProvider.getInitialSnapshot()).thenReturn(SnapshotMockUtils.createSnapshot()); TipsViewModel tipsViewModel = new TipsViewModel(); txRequester = new TransactionRequester(tangle, snapshotProvider); - txValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, txRequester, new MainnetConfig(), txSolidifier); + txValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, txRequester, new MainnetConfig()); txValidator.setMwm(false, MAINNET_MWM); txValidator.init(); } @@ -82,7 +78,7 @@ public void setUpEach() { public void testMinMwm() { ProtocolConfig protocolConfig = mock(ProtocolConfig.class); when(protocolConfig.getMwm()).thenReturn(5); - TransactionValidator transactionValidator = new TransactionValidator(null, null, null, null, protocolConfig, txSolidifier); + TransactionValidator transactionValidator = new TransactionValidator(null, null, null, null, protocolConfig); assertEquals("Expected testnet minimum minWeightMagnitude", 13, transactionValidator.getMinWeightMagnitude()); } From f5de8fe7132a387022625bd5efba1ce7f2e723d1 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Wed, 4 Dec 2019 11:25:37 -0700 Subject: [PATCH 37/48] Ensure solidTip is solid before broadcasting --- .../java/com/iota/iri/network/pipeline/SolidifyStage.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java index 1dee56838d..02ef6e405f 100644 --- a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java @@ -70,9 +70,11 @@ public ProcessingContext process(ProcessingContext ctx){ Hash tipHash = tipsViewModel.getRandomSolidTipHash(); if(tipHash != null) { TransactionViewModel solidTip = fromHash(tangle, tipHash); - ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); - ctx.setPayload(new BroadcastPayload(payload.getOriginNeighbor(), solidTip)); - return ctx; + if(solidTip.isSolid()) { + ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); + ctx.setPayload(new BroadcastPayload(payload.getOriginNeighbor(), solidTip)); + return ctx; + } } ctx.setNextStage(TransactionProcessingPipeline.Stage.FINISH); From 30f226eda9c657622203920a35ad481f69975fff Mon Sep 17 00:00:00 2001 From: DyrellC Date: Wed, 4 Dec 2019 15:20:39 -0700 Subject: [PATCH 38/48] Update tests, only broadcast solid transactions from solidify stage --- .../iri/network/pipeline/SolidifyStage.java | 43 +++++++++++++------ .../network/pipeline/SolidifyStageTest.java | 4 +- .../impl/TransactionSolidifierImplTest.java | 6 ++- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java index 02ef6e405f..a56616c96b 100644 --- a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java @@ -1,5 +1,6 @@ package com.iota.iri.network.pipeline; +import com.google.common.annotations.VisibleForTesting; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; @@ -26,6 +27,7 @@ public class SolidifyStage implements Stage { private TransactionValidator txValidator; private TipsViewModel tipsViewModel; private Tangle tangle; + private TransactionViewModel tip; /** * Constructor for the {@link SolidifyStage}. @@ -67,18 +69,7 @@ public ProcessingContext process(ProcessingContext ctx){ txSolidifier.addToSolidificationQueue(tvm.getHash()); - Hash tipHash = tipsViewModel.getRandomSolidTipHash(); - if(tipHash != null) { - TransactionViewModel solidTip = fromHash(tangle, tipHash); - if(solidTip.isSolid()) { - ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); - ctx.setPayload(new BroadcastPayload(payload.getOriginNeighbor(), solidTip)); - return ctx; - } - } - - ctx.setNextStage(TransactionProcessingPipeline.Stage.FINISH); - return ctx; + return broadcastTip(ctx, payload); }catch (Exception e){ log.error("Failed to process transaction for solidification", e); ctx.setNextStage(TransactionProcessingPipeline.Stage.ABORT); @@ -86,4 +77,32 @@ public ProcessingContext process(ProcessingContext ctx){ } } + + private ProcessingContext broadcastTip(ProcessingContext ctx, SolidifyPayload payload) throws Exception{ + if(tip == null) { + Hash tipHash = tipsViewModel.getRandomSolidTipHash(); + + if (tipHash == null) { + ctx.setNextStage(TransactionProcessingPipeline.Stage.FINISH); + return ctx; + } + + tip = fromHash(tangle, tipHash); + } + + if(tip.isSolid()) { + ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); + ctx.setPayload(new BroadcastPayload(payload.getOriginNeighbor(), tip)); + return ctx; + } + + ctx.setNextStage(TransactionProcessingPipeline.Stage.FINISH); + return ctx; + } + + @VisibleForTesting + void injectTip(TransactionViewModel tvm) throws Exception { + tip = tvm; + tip.updateSolid(true); + } } diff --git a/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java b/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java index 17b35f8ab1..43830c6d8f 100644 --- a/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java @@ -3,6 +3,7 @@ import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +import com.iota.iri.model.persistables.Transaction; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Tangle; @@ -82,12 +83,13 @@ public void quickSetSolidTransactionIsBroadcast() throws Exception{ public void unsolidTransactionBroadcastsRandomSolidTip() throws Exception{ Mockito.when(tvm.isSolid()).thenReturn(false); Mockito.when(transactionValidator.quickSetSolid(tvm)).thenReturn(false); - Mockito.when(tipsViewModel.getRandomSolidTipHash()).thenReturn(tipHash); + TransactionViewModel tip = new TransactionViewModel(new Transaction(), tipHash); SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, transactionValidator, tipsViewModel, tangle); SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); ProcessingContext ctx = new ProcessingContext(solidifyPayload); + solidifyStage.injectTip(tip); solidifyStage.process(ctx); Thread.sleep(100); diff --git a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java index 3ad4bf8d63..873e039159 100644 --- a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java +++ b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java @@ -90,12 +90,14 @@ public void verifyTxIsNotSolid() throws Exception { @Test public void getSolidificationQueue() throws Exception { - TransactionViewModel tx = getTxWithBranchAndTrunk(); + TransactionViewModel mainTx = getTxWithBranchAndTrunk(); for(int i = 0; i < 10; i++) { + TransactionViewModel tx = getTxWithBranchAndTrunk(); txSolidifier.addToSolidificationQueue(tx.getHash()); } + txSolidifier.addToSolidificationQueue(mainTx.getHash()); assertTrue("Expected transaction to be present in the solidification queue", - txSolidifier.getSolidificationQueue().contains(tx.getHash())); + txSolidifier.getSolidificationQueue().contains(mainTx.getHash())); } @Test From 303648baa61e7660174b0e8a3e173dea5bae479b Mon Sep 17 00:00:00 2001 From: DyrellC Date: Thu, 5 Dec 2019 15:03:08 -0700 Subject: [PATCH 39/48] Move validator test to appropriate package --- .../com/iota/iri/service/validation/TransactionValidator.java | 4 ++-- .../{ => service/validation}/TransactionValidatorTest.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) rename src/test/java/com/iota/iri/{ => service/validation}/TransactionValidatorTest.java (98%) diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index d190baa183..2a205ddcaf 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -92,7 +92,7 @@ public void init() { * Set the Minimum Weight Magnitude for validation checks. */ @VisibleForTesting - public void setMwm(boolean testnet, int mwm) { + void setMwm(boolean testnet, int mwm) { minWeightMagnitude = mwm; //lowest allowed MWM encoded in 46 bytes. @@ -382,7 +382,7 @@ private boolean checkApproovee(TransactionViewModel approovee) throws Exception * Exclusively used in the {@link TansactionValidatorTest} */ @VisibleForTesting - public boolean isNewSolidTxSetsEmpty () { + boolean isNewSolidTxSetsEmpty () { return newSolidTransactionsOne.isEmpty() && newSolidTransactionsTwo.isEmpty(); } diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java similarity index 98% rename from src/test/java/com/iota/iri/TransactionValidatorTest.java rename to src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java index 7314a3d8cc..8af0eeb8d7 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java @@ -1,5 +1,6 @@ -package com.iota.iri; +package com.iota.iri.service.validation; +import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.conf.ProtocolConfig; import com.iota.iri.controllers.TipsViewModel; From 7e36e1c0055f84c618e80083e04164159cbe493f Mon Sep 17 00:00:00 2001 From: DyrellC Date: Thu, 5 Dec 2019 16:17:17 -0700 Subject: [PATCH 40/48] Downgrade log error to log info --- .../iri/service/validation/impl/TransactionSolidifierImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index 1052be14d6..924d025ebf 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -256,7 +256,7 @@ private void addToBroadcastQueue(TransactionViewModel tvm) { transactionsToBroadcast.put(tvm); } catch(Exception e){ - log.error("Error placing transaction into broadcast queue", e); + log.info("Error placing transaction into broadcast queue: " + e.getMessage()); } } From eb53b2fc6d15c947114c7ef1254049a8c8ae41da Mon Sep 17 00:00:00 2001 From: DyrellC Date: Tue, 10 Dec 2019 11:48:50 -0700 Subject: [PATCH 41/48] Remove unnecessary solidity propogation thread in validator --- src/main/java/com/iota/iri/Iota.java | 3 - .../validation/TransactionValidator.java | 144 ------------------ .../validation/TransactionValidatorTest.java | 91 ----------- 3 files changed, 238 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index e383a4869d..9c32faa135 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -190,8 +190,6 @@ public void init() throws Exception { tangle.clearMetadata(com.iota.iri.model.persistables.Transaction.class); } - transactionValidator.init(); - txPipeline.start(); neighborRouter.start(); tipsRequester.start(); @@ -257,7 +255,6 @@ public void shutdown() throws Exception { tipsRequester.shutdown(); txPipeline.shutdown(); neighborRouter.shutdown(); - transactionValidator.shutdown(); tangle.shutdown(); // free the resources of the snapshot provider last because all other instances need it diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index 2a205ddcaf..0a59441c50 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -7,23 +7,15 @@ import com.iota.iri.crypto.Curl; import com.iota.iri.crypto.Sponge; import com.iota.iri.crypto.SpongeFactory; -import com.iota.iri.model.Hash; import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; import static com.iota.iri.controllers.TransactionViewModel.*; public class TransactionValidator { - private static final Logger log = LoggerFactory.getLogger(TransactionValidator.class); private static final int TESTNET_MWM_CAP = 13; - public static final int SOLID_SLEEP_TIME = 500; private final Tangle tangle; private final SnapshotProvider snapshotProvider; @@ -33,24 +25,6 @@ public class TransactionValidator { private static final long MAX_TIMESTAMP_FUTURE = 2L * 60L * 60L; private static final long MAX_TIMESTAMP_FUTURE_MS = MAX_TIMESTAMP_FUTURE * 1_000L; - /////////////////////////////////fields for solidification thread////////////////////////////////////// - - private Thread newSolidThread; - - /** - * If true use {@link #newSolidTransactionsOne} while solidifying. Else use {@link #newSolidTransactionsTwo}. - */ - private final AtomicBoolean useFirst = new AtomicBoolean(true); - /** - * Is {@link #newSolidThread} shutting down - */ - private final AtomicBoolean shuttingDown = new AtomicBoolean(false); - /** - * mutex for solidification - */ - private final Object cascadeSync = new Object(); - private final Set newSolidTransactionsOne = new LinkedHashSet<>(); - private final Set newSolidTransactionsTwo = new LinkedHashSet<>(); /** * Constructor for Tangle Validator @@ -69,25 +43,9 @@ public TransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, Ti this.snapshotProvider = snapshotProvider; this.tipsViewModel = tipsViewModel; this.transactionRequester = transactionRequester; - this.newSolidThread = new Thread(spawnSolidTransactionsPropagation(), "Solid TX cascader"); setMwm(protocolConfig.isTestnet(), protocolConfig.getMwm()); } - /** - * Does two things: - *

    - *
  1. Sets the minimum weight magnitude (MWM). POW on a transaction is validated by counting a certain - * number of consecutive 9s in the end of the transaction hash. The number of 9s is the MWM.
    2. - *
  2. Starts the transaction solidification thread.
    3. - *
- * - * - * @see #spawnSolidTransactionsPropagation() - */ - public void init() { - newSolidThread.start(); - } - /** * Set the Minimum Weight Magnitude for validation checks. */ @@ -101,16 +59,6 @@ void setMwm(boolean testnet, int mwm) { } } - /** - * Shutdown roots to tip solidification thread - * @throws InterruptedException - * @see #spawnSolidTransactionsPropagation() - */ - public void shutdown() throws InterruptedException { - shuttingDown.set(true); - newSolidThread.join(); - } - /** * @return the minimal number of trailing 9s that have to be present at the end of the transaction hash * in order to validate that sufficient proof of work has been done @@ -212,75 +160,6 @@ public TransactionViewModel validateBytes(final byte[] bytes, int minWeightMagni return transactionViewModel; } - - public void addSolidTransaction(Hash hash) { - synchronized (cascadeSync) { - if (useFirst.get()) { - newSolidTransactionsOne.add(hash); - } else { - newSolidTransactionsTwo.add(hash); - } - } - } - - /** - * Creates a runnable that runs {@link #propagateSolidTransactions()} in a loop every {@value #SOLID_SLEEP_TIME} ms - * @return runnable that is not started - */ - private Runnable spawnSolidTransactionsPropagation() { - return () -> { - while(!shuttingDown.get()) { - propagateSolidTransactions(); - try { - Thread.sleep(SOLID_SLEEP_TIME); - } catch (InterruptedException e) { - // Ignoring InterruptedException. Do not use Thread.currentThread().interrupt() here. - log.error("Thread was interrupted: ", e); - } - } - }; - } - - /** - * Iterates over all currently known solid transactions. For each solid transaction, we find - * its children (approvers) and try to quickly solidify them with {@link #quietQuickSetSolid}. - * If we manage to solidify the transactions, we add them to the solidification queue for a traversal by a later run. - */ - public void propagateSolidTransactions() { - Set newSolidHashes = new HashSet<>(); - useFirst.set(!useFirst.get()); - //synchronized to make sure no one is changing the newSolidTransactions collections during addAll - synchronized (cascadeSync) { - //We are using a collection that doesn't get updated by other threads - if (useFirst.get()) { - newSolidHashes.addAll(newSolidTransactionsTwo); - newSolidTransactionsTwo.clear(); - } else { - newSolidHashes.addAll(newSolidTransactionsOne); - newSolidTransactionsOne.clear(); - } - } - Iterator cascadeIterator = newSolidHashes.iterator(); - while(cascadeIterator.hasNext() && !shuttingDown.get()) { - try { - Hash hash = cascadeIterator.next(); - TransactionViewModel transaction = fromHash(tangle, hash); - Set approvers = transaction.getApprovers(tangle).getHashes(); - for(Hash h: approvers) { - TransactionViewModel tx = fromHash(tangle, h); - if(quietQuickSetSolid(tx)) { - tx.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); - tipsViewModel.setSolid(h); - addSolidTransaction(h); - } - } - } catch (Exception e) { - log.error("Error while propagating solidity upwards", e); - } - } - } - - /** * Updates a transaction after it was stored in the tangle. Tells the node to not request the transaction anymore, * to update the live tips accordingly, and attempts to quickly solidify the transaction. @@ -316,21 +195,6 @@ public void updateStatus(TransactionViewModel transactionViewModel) throws Excep if(quickSetSolid(transactionViewModel)) { transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); tipsViewModel.setSolid(transactionViewModel.getHash()); - addSolidTransaction(transactionViewModel.getHash()); - } - } - - /** - * Perform a {@link #quickSetSolid} while capturing and logging errors - * @param transactionViewModel transaction we try to solidify. - * @return true if we managed to solidify, else false. - */ - private boolean quietQuickSetSolid(TransactionViewModel transactionViewModel) { - try { - return quickSetSolid(transactionViewModel); - } catch (Exception e) { - log.error(e.getMessage(), e); - return false; } } @@ -378,14 +242,6 @@ private boolean checkApproovee(TransactionViewModel approovee) throws Exception return approovee.isSolid(); } - /** - * Exclusively used in the {@link TansactionValidatorTest} - */ - @VisibleForTesting - boolean isNewSolidTxSetsEmpty () { - return newSolidTransactionsOne.isEmpty() && newSolidTransactionsTwo.isEmpty(); - } - /** * Thrown if transaction fails {@link #hasInvalidTimestamp} check. */ diff --git a/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java b/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java index 8af0eeb8d7..484b888061 100644 --- a/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java @@ -1,16 +1,12 @@ package com.iota.iri.service.validation; -import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.conf.ProtocolConfig; import com.iota.iri.controllers.TipsViewModel; -import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.crypto.SpongeFactory; -import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; -import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Converter; @@ -21,12 +17,8 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -import static com.iota.iri.TransactionTestUtils.getTransactionHash; import static com.iota.iri.TransactionTestUtils.getTransactionTrits; -import static com.iota.iri.TransactionTestUtils.getTransactionTritsWithTrunkAndBranch; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -72,7 +64,6 @@ public void setUpEach() { txRequester = new TransactionRequester(tangle, snapshotProvider); txValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, txRequester, new MainnetConfig()); txValidator.setMwm(false, MAINNET_MWM); - txValidator.init(); } @Test @@ -104,86 +95,4 @@ public void validateBytesWithNewCurl() { Converter.bytes(trits, 0, bytes, 0, trits.length); txValidator.validateBytes(bytes, txValidator.getMinWeightMagnitude(), SpongeFactory.create(SpongeFactory.Mode.CURLP81)); } - - - @Test - public void addSolidTransactionWithoutErrors() { - byte[] trits = getTransactionTrits(); - Converter.copyTrits(0, trits, 0, trits.length); - txValidator.addSolidTransaction(TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); - } - - - - @Test - public void testTransactionPropagation() throws Exception { - TransactionViewModel leftChildLeaf = TransactionTestUtils.createTransactionWithTrytes("CHILDTX"); - leftChildLeaf.updateSolid(true); - leftChildLeaf.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel rightChildLeaf = TransactionTestUtils.createTransactionWithTrytes("CHILDTWOTX"); - rightChildLeaf.updateSolid(true); - rightChildLeaf.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel parent = TransactionTestUtils.createTransactionWithTrunkAndBranch("PARENT", - leftChildLeaf.getHash(), rightChildLeaf.getHash()); - parent.updateSolid(false); - parent.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel parentSibling = TransactionTestUtils.createTransactionWithTrytes("PARENTLEAF"); - parentSibling.updateSolid(true); - parentSibling.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel grandParent = TransactionTestUtils.createTransactionWithTrunkAndBranch("GRANDPARENT", parent.getHash(), - parentSibling.getHash()); - grandParent.updateSolid(false); - grandParent.store(tangle, snapshotProvider.getInitialSnapshot()); - - txValidator.addSolidTransaction(leftChildLeaf.getHash()); - while (!txValidator.isNewSolidTxSetsEmpty()) { - txValidator.propagateSolidTransactions(); - } - - parent = TransactionViewModel.fromHash(tangle, parent.getHash()); - assertTrue("Parent tx was expected to be solid", parent.isSolid()); - grandParent = TransactionViewModel.fromHash(tangle, grandParent.getHash()); - assertTrue("Grandparent was expected to be solid", grandParent.isSolid()); - } - - @Test - public void testTransactionPropagationFailure() throws Exception { - TransactionViewModel leftChildLeaf = new TransactionViewModel(getTransactionTrits(), getTransactionHash()); - leftChildLeaf.updateSolid(true); - leftChildLeaf.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel rightChildLeaf = new TransactionViewModel(getTransactionTrits(), getTransactionHash()); - rightChildLeaf.updateSolid(true); - rightChildLeaf.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel parent = new TransactionViewModel(getTransactionTritsWithTrunkAndBranch(leftChildLeaf.getHash(), - rightChildLeaf.getHash()), getTransactionHash()); - parent.updateSolid(false); - parent.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel parentSibling = new TransactionViewModel(getTransactionTrits(), getTransactionHash()); - parentSibling.updateSolid(false); - parentSibling.store(tangle, snapshotProvider.getInitialSnapshot()); - - TransactionViewModel grandParent = new TransactionViewModel(getTransactionTritsWithTrunkAndBranch(parent.getHash(), - parentSibling.getHash()), getTransactionHash()); - grandParent.updateSolid(false); - grandParent.store(tangle, snapshotProvider.getInitialSnapshot()); - - txValidator.addSolidTransaction(leftChildLeaf.getHash()); - while (!txValidator.isNewSolidTxSetsEmpty()) { - txValidator.propagateSolidTransactions(); - } - - parent = TransactionViewModel.fromHash(tangle, parent.getHash()); - assertTrue("Parent tx was expected to be solid", parent.isSolid()); - grandParent = TransactionViewModel.fromHash(tangle, grandParent.getHash()); - assertFalse("GrandParent tx was expected to be not solid", grandParent.isSolid()); - } - - } From 0409d3e0f8abe58ca48e938c8dd87c90bcd9c383 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Tue, 10 Dec 2019 12:01:18 -0700 Subject: [PATCH 42/48] only import used parts of junit --- .../iri/service/validation/TransactionValidatorTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java b/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java index 484b888061..8ef6ab2245 100644 --- a/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java @@ -11,7 +11,11 @@ import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Converter; -import org.junit.*; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.Rule; +import org.junit.AfterClass; import org.junit.rules.TemporaryFolder; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; From 1f40bccbb393334bce0f916cb66ec072e1f43dc5 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 23 Dec 2019 22:30:02 -0700 Subject: [PATCH 43/48] Initial commit - Milestone Stage --- src/main/java/com/iota/iri/Iota.java | 19 +- .../iota/iri/MainInjectionConfiguration.java | 59 +- .../iota/iri/conf/SolidificationConfig.java | 4 + .../iri/controllers/MilestoneViewModel.java | 2 +- .../iri/controllers/TransactionViewModel.java | 6 +- .../NetworkInjectionConfiguration.java | 16 +- .../iri/network/impl/TipsRequesterImpl.java | 12 +- .../network/pipeline/MilestonePayload.java | 56 ++ .../iri/network/pipeline/MilestoneStage.java | 107 +++ .../iri/network/pipeline/ReceivedStage.java | 35 +- .../iota/iri/network/pipeline/ReplyStage.java | 26 +- .../iri/network/pipeline/SolidifyStage.java | 34 +- .../TransactionProcessingPipeline.java | 24 +- .../TransactionProcessingPipelineImpl.java | 47 +- src/main/java/com/iota/iri/service/API.java | 28 +- .../milestone/LatestMilestoneTracker.java | 106 --- .../LatestSolidMilestoneTracker.java | 42 - .../service/milestone/MilestoneService.java | 4 +- .../milestone/MilestoneSolidifier.java | 64 ++ .../impl/LatestMilestoneTrackerImpl.java | 390 --------- .../impl/LatestSolidMilestoneTrackerImpl.java | 471 ----------- .../impl/MilestoneSolidifierImpl.java | 771 ++++++++++++------ .../snapshot/LocalSnapshotManager.java | 8 +- .../iri/service/snapshot/SnapshotService.java | 16 +- .../impl/LocalSnapshotManagerImpl.java | 28 +- .../snapshot/impl/SnapshotServiceImpl.java | 16 +- .../impl/EntryPointSelectorImpl.java | 12 +- .../validation/TransactionSolidifier.java | 42 + .../validation/TransactionValidator.java | 82 -- .../impl/TransactionSolidifierImpl.java | 150 +++- .../iri/MainInjectionConfigurationTest.java | 12 - .../NetworkInjectionConfigurationTest.java | 6 +- .../network/pipeline/ReceivedStageTest.java | 16 +- .../iri/network/pipeline/ReplyStageTest.java | 10 +- .../network/pipeline/SolidifyStageTest.java | 17 +- .../TransactionProcessingPipelineTest.java | 96 ++- .../java/com/iota/iri/service/APITest.java | 6 +- .../com/iota/iri/service/ApiCallTest.java | 2 +- .../impl/LocalSnapshotManagerImplTest.java | 32 +- .../impl/EntryPointSelectorImplTest.java | 10 +- .../impl/TransactionSolidifierImplTest.java | 9 +- 41 files changed, 1312 insertions(+), 1581 deletions(-) create mode 100644 src/main/java/com/iota/iri/network/pipeline/MilestonePayload.java create mode 100644 src/main/java/com/iota/iri/network/pipeline/MilestoneStage.java delete mode 100644 src/main/java/com/iota/iri/service/milestone/LatestMilestoneTracker.java delete mode 100644 src/main/java/com/iota/iri/service/milestone/LatestSolidMilestoneTracker.java delete mode 100644 src/main/java/com/iota/iri/service/milestone/impl/LatestMilestoneTrackerImpl.java delete mode 100644 src/main/java/com/iota/iri/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 9c32faa135..e53b475868 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -15,8 +15,6 @@ import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.ledger.LedgerService; -import com.iota.iri.service.milestone.LatestMilestoneTracker; -import com.iota.iri.service.milestone.LatestSolidMilestoneTracker; import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.milestone.SeenMilestonesRetriever; @@ -88,10 +86,6 @@ public class Iota { public final MilestoneService milestoneService; - public final LatestMilestoneTracker latestMilestoneTracker; - - public final LatestSolidMilestoneTracker latestSolidMilestoneTracker; - public final SeenMilestonesRetriever seenMilestonesRetriever; public final LedgerService ledgerService; @@ -120,7 +114,7 @@ public class Iota { * @param configuration Information about how this node will be configured. * */ - public Iota(IotaConfig configuration, SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, LocalSnapshotManager localSnapshotManager, MilestoneService milestoneService, LatestMilestoneTracker latestMilestoneTracker, LatestSolidMilestoneTracker latestSolidMilestoneTracker, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector, TransactionSolidifier transactionSolidifier) { + public Iota(IotaConfig configuration, SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, LocalSnapshotManager localSnapshotManager, MilestoneService milestoneService, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector, TransactionSolidifier transactionSolidifier) { this.configuration = configuration; this.ledgerService = ledgerService; @@ -130,8 +124,6 @@ public Iota(IotaConfig configuration, SpentAddressesProvider spentAddressesProvi this.snapshotService = snapshotService; this.localSnapshotManager = localSnapshotManager; this.milestoneService = milestoneService; - this.latestMilestoneTracker = latestMilestoneTracker; - this.latestSolidMilestoneTracker = latestSolidMilestoneTracker; this.seenMilestonesRetriever = seenMilestonesRetriever; this.milestoneSolidifier = milestoneSolidifier; this.transactionPruner = transactionPruner; @@ -157,7 +149,6 @@ private void initDependencies() throws SnapshotException, SpentAddressesExceptio boolean assertSpentAddressesExistence = !configuration.isTestnet() && snapshotProvider.getInitialSnapshot().getIndex() != configuration.getMilestoneStartIndex(); spentAddressesProvider.init(assertSpentAddressesExistence); - latestMilestoneTracker.init(); seenMilestonesRetriever.init(); if (transactionPruner != null) { transactionPruner.init(); @@ -194,14 +185,12 @@ public void init() throws Exception { neighborRouter.start(); tipsRequester.start(); - latestMilestoneTracker.start(); - latestSolidMilestoneTracker.start(); seenMilestonesRetriever.start(); - milestoneSolidifier.start(); transactionSolidifier.start(); + milestoneSolidifier.start(); if (localSnapshotManager != null) { - localSnapshotManager.start(latestMilestoneTracker); + localSnapshotManager.start(milestoneSolidifier); } if (transactionPruner != null) { transactionPruner.start(); @@ -242,8 +231,6 @@ public void shutdown() throws Exception { milestoneSolidifier.shutdown(); transactionSolidifier.shutdown(); seenMilestonesRetriever.shutdown(); - latestSolidMilestoneTracker.shutdown(); - latestMilestoneTracker.shutdown(); if (transactionPruner != null) { transactionPruner.shutdown(); diff --git a/src/main/java/com/iota/iri/MainInjectionConfiguration.java b/src/main/java/com/iota/iri/MainInjectionConfiguration.java index c22e4879f5..90abd61911 100644 --- a/src/main/java/com/iota/iri/MainInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/MainInjectionConfiguration.java @@ -18,16 +18,10 @@ import com.iota.iri.service.API; import com.iota.iri.service.ledger.LedgerService; import com.iota.iri.service.ledger.impl.LedgerServiceImpl; -import com.iota.iri.service.milestone.LatestMilestoneTracker; -import com.iota.iri.service.milestone.LatestSolidMilestoneTracker; import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.milestone.SeenMilestonesRetriever; -import com.iota.iri.service.milestone.impl.LatestMilestoneTrackerImpl; -import com.iota.iri.service.milestone.impl.LatestSolidMilestoneTrackerImpl; -import com.iota.iri.service.milestone.impl.MilestoneServiceImpl; -import com.iota.iri.service.milestone.impl.MilestoneSolidifierImpl; -import com.iota.iri.service.milestone.impl.SeenMilestonesRetrieverImpl; +import com.iota.iri.service.milestone.impl.*; import com.iota.iri.service.snapshot.LocalSnapshotManager; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.SnapshotService; @@ -117,33 +111,12 @@ LedgerService provideLedgerService(Tangle tangle, SnapshotProvider snapshotProvi return new LedgerServiceImpl(tangle, snapshotProvider, snapshotService, milestoneService, spentAddressesService, bundleValidator); } - @Singleton - @Provides - LatestMilestoneTracker provideLatestMilestoneTracker(Tangle tangle, SnapshotProvider snapshotProvider, MilestoneService milestoneService, MilestoneSolidifier milestoneSolidifier) { - return new LatestMilestoneTrackerImpl(tangle, snapshotProvider, milestoneService, milestoneSolidifier, configuration); - } - - @Singleton - @Provides - LatestSolidMilestoneTracker provideLatestSolidMilestoneTracker(Tangle tangle, SnapshotProvider snapshotProvider, - MilestoneService milestoneService, LedgerService ledgerService, - LatestMilestoneTracker latestMilestoneTracker, TransactionRequester transactionRequester) { - return new LatestSolidMilestoneTrackerImpl(tangle, snapshotProvider, milestoneService, ledgerService, - latestMilestoneTracker, transactionRequester, configuration); - } - @Singleton @Provides SeenMilestonesRetriever provideSeenMilestonesRetriever(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester) { return new SeenMilestonesRetrieverImpl(tangle, snapshotProvider, transactionRequester); } - @Singleton - @Provides - MilestoneSolidifier provideMilestoneSolidifier(SnapshotProvider snapshotProvider, TransactionSolidifier transactionSolidifier) { - return new MilestoneSolidifierImpl(snapshotProvider, transactionSolidifier); - } - @Singleton @Provides TransactionPruner provideTransactionPruner(Tangle tangle, SnapshotProvider snapshotProvider, SpentAddressesService spentAddressesService, SpentAddressesProvider spentAddressesProvider, TipsViewModel tipsViewModel) { @@ -168,16 +141,16 @@ TransactionValidator provideTransactionValidator(Tangle tangle, SnapshotProvider @Singleton @Provides - TransactionSolidifier provideTransactionSolidifier(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester){ - return new TransactionSolidifierImpl(tangle, snapshotProvider, transactionRequester); + TransactionSolidifier provideTransactionSolidifier(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester, TipsViewModel tipsViewModel){ + return new TransactionSolidifierImpl(tangle, snapshotProvider, transactionRequester, tipsViewModel, configuration.getCoordinator()); } @Singleton @Provides TipSelector provideTipSelector(Tangle tangle, SnapshotProvider snapshotProvider, - LatestMilestoneTracker latestMilestoneTracker, LedgerService ledgerService) { + MilestoneSolidifier milestoneSolidifier, LedgerService ledgerService) { EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, - latestMilestoneTracker); + milestoneSolidifier); RatingCalculator ratingCalculator = new CumulativeWeightCalculator(tangle, snapshotProvider); TailFinder tailFinder = new TailFinderImpl(tangle); Walker walker = new WalkerAlpha(tailFinder, tangle, new SecureRandom(), configuration); @@ -187,8 +160,8 @@ TipSelector provideTipSelector(Tangle tangle, SnapshotProvider snapshotProvider, @Singleton @Provides - Iota provideIota(SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, @Nullable LocalSnapshotManager localSnapshotManager, MilestoneService milestoneService, LatestMilestoneTracker latestMilestoneTracker, LatestSolidMilestoneTracker latestSolidMilestoneTracker, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, @Nullable TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector, TransactionSolidifier transactionSolidifier) { - return new Iota(configuration, spentAddressesProvider, spentAddressesService, snapshotProvider, snapshotService, localSnapshotManager, milestoneService, latestMilestoneTracker, latestSolidMilestoneTracker, seenMilestonesRetriever, ledgerService, transactionPruner, milestoneSolidifier, bundleValidator, tangle, transactionValidator, transactionRequester, neighborRouter, transactionProcessingPipeline, tipsRequester, tipsViewModel, tipsSelector, transactionSolidifier); + Iota provideIota(SpentAddressesProvider spentAddressesProvider, SpentAddressesService spentAddressesService, SnapshotProvider snapshotProvider, SnapshotService snapshotService, @Nullable LocalSnapshotManager localSnapshotManager, MilestoneService milestoneService, SeenMilestonesRetriever seenMilestonesRetriever, LedgerService ledgerService, @Nullable TransactionPruner transactionPruner, MilestoneSolidifier milestoneSolidifier, BundleValidator bundleValidator, Tangle tangle, TransactionValidator transactionValidator, TransactionRequester transactionRequester, NeighborRouter neighborRouter, TransactionProcessingPipeline transactionProcessingPipeline, TipsRequester tipsRequester, TipsViewModel tipsViewModel, TipSelector tipsSelector, TransactionSolidifier transactionSolidifier) { + return new Iota(configuration, spentAddressesProvider, spentAddressesService, snapshotProvider, snapshotService, localSnapshotManager, milestoneService, seenMilestonesRetriever, ledgerService, transactionPruner, milestoneSolidifier, bundleValidator, tangle, transactionValidator, transactionRequester, neighborRouter, transactionProcessingPipeline, tipsRequester, tipsViewModel, tipsSelector, transactionSolidifier); } @Singleton @@ -200,11 +173,19 @@ IXI provideIxi(Iota iota) { @Singleton @Provides API provideApi(IXI ixi, TransactionRequester transactionRequester, - SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, - SnapshotProvider snapshotProvider, LedgerService ledgerService, NeighborRouter neighborRouter, TipSelector tipsSelector, - TipsViewModel tipsViewModel, TransactionValidator transactionValidator, - LatestMilestoneTracker latestMilestoneTracker, TransactionProcessingPipeline txPipeline) { - return new API(configuration, ixi, transactionRequester, spentAddressesService, tangle, bundleValidator, snapshotProvider, ledgerService, neighborRouter, tipsSelector, tipsViewModel, transactionValidator, latestMilestoneTracker, txPipeline); + SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, + SnapshotProvider snapshotProvider, LedgerService ledgerService, NeighborRouter neighborRouter, TipSelector tipsSelector, + TipsViewModel tipsViewModel, TransactionValidator transactionValidator, + MilestoneSolidifier milestoneSolidifier, TransactionProcessingPipeline txPipeline, + TransactionSolidifier transactionSolidifier) { + return new API(configuration, ixi, transactionRequester, spentAddressesService, tangle, bundleValidator, snapshotProvider, ledgerService, neighborRouter, tipsSelector, tipsViewModel, transactionValidator, milestoneSolidifier, txPipeline, transactionSolidifier); + } + + @Singleton + @Provides + MilestoneSolidifier provideMilestoneSolidifier(TransactionSolidifier transactionSolidifier, Tangle tangle, SnapshotProvider snapshotProvider, LedgerService ledgerService, TransactionRequester transactionRequester, MilestoneService milestoneService){ + return new MilestoneSolidifierImpl(transactionSolidifier,tangle,snapshotProvider, ledgerService, + transactionRequester, milestoneService, configuration); } @Override diff --git a/src/main/java/com/iota/iri/conf/SolidificationConfig.java b/src/main/java/com/iota/iri/conf/SolidificationConfig.java index 495b11cd70..2492cf98fc 100644 --- a/src/main/java/com/iota/iri/conf/SolidificationConfig.java +++ b/src/main/java/com/iota/iri/conf/SolidificationConfig.java @@ -1,5 +1,7 @@ package com.iota.iri.conf; +import com.iota.iri.model.Hash; + /** * * Configurations that should be used for the solidification processes. @@ -13,6 +15,8 @@ public interface SolidificationConfig extends Config { */ boolean isPrintSyncProgressEnabled(); + Hash getCoordinator(); + /** * Field descriptions */ diff --git a/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java b/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java index 9f35a9724e..b1dfe078cd 100644 --- a/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java +++ b/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java @@ -13,7 +13,7 @@ /** * Acts as a controller interface for a {@link Milestone} hash object. This controller is used by the - * {@link com.iota.iri.MilestoneTracker} to manipulate a {@link Milestone} object. + * {@link com.iota.iri.service.milestone.MilestoneSolidifier} to manipulate a {@link Milestone} object. */ public class MilestoneViewModel { private final Milestone milestone; diff --git a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java index e504d90f73..4047054b64 100644 --- a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java +++ b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java @@ -760,9 +760,9 @@ public void setSnapshot(Tangle tangle, Snapshot initialSnapshot, final int index /** * This method sets the {@link Transaction#milestone} flag. * - * It gets automatically called by the {@link com.iota.iri.service.milestone.LatestMilestoneTracker} and marks transactions that represent a - * milestone accordingly. It first checks if the {@link Transaction#milestone} flag has changed and if so, it issues - * a database update. + * It gets automatically called by the {@link com.iota.iri.service.milestone.MilestoneSolidifier} and marks + * transactions that represent a milestone accordingly. It first checks if the {@link Transaction#milestone} flag + * has changed and if so, it issues a database update. * * @param tangle Tangle instance which acts as a database interface * @param initialSnapshot the snapshot representing the starting point of our ledger diff --git a/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java b/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java index d4caa618e5..ee247e1bb7 100644 --- a/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/network/NetworkInjectionConfiguration.java @@ -3,6 +3,8 @@ import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.IotaConfig; @@ -10,7 +12,6 @@ import com.iota.iri.network.impl.TipsRequesterImpl; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.network.pipeline.TransactionProcessingPipelineImpl; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; @@ -38,18 +39,21 @@ TransactionRequester provideTransactionRequester(Tangle tangle, SnapshotProvider @Singleton @Provides - TipsRequester provideTipsRequester(NeighborRouter neighborRouter, Tangle tangle, LatestMilestoneTracker latestMilestoneTracker, TransactionRequester txRequester) { - return new TipsRequesterImpl(neighborRouter, tangle, latestMilestoneTracker, txRequester); + TipsRequester provideTipsRequester(NeighborRouter neighborRouter, Tangle tangle, + MilestoneSolidifier milestoneSolidifier, TransactionRequester txRequester) { + return new TipsRequesterImpl(neighborRouter, tangle, milestoneSolidifier, txRequester); } @Singleton @Provides TransactionProcessingPipeline provideTransactionProcessingPipeline(NeighborRouter neighborRouter, TransactionValidator txValidator, Tangle tangle, SnapshotProvider snapshotProvider, - TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, - TransactionRequester transactionRequester, TransactionSolidifier transactionSolidifier) { + TipsViewModel tipsViewModel, TransactionRequester transactionRequester, + TransactionSolidifier transactionSolidifier, MilestoneService milestoneService, + MilestoneSolidifier milestoneSolidifier) { return new TransactionProcessingPipelineImpl(neighborRouter, configuration, txValidator, tangle, - snapshotProvider, tipsViewModel, latestMilestoneTracker, transactionRequester, transactionSolidifier); + snapshotProvider, tipsViewModel, transactionRequester, transactionSolidifier, + milestoneService, milestoneSolidifier); } @Singleton diff --git a/src/main/java/com/iota/iri/network/impl/TipsRequesterImpl.java b/src/main/java/com/iota/iri/network/impl/TipsRequesterImpl.java index 58d0405e15..19697ffa5e 100644 --- a/src/main/java/com/iota/iri/network/impl/TipsRequesterImpl.java +++ b/src/main/java/com/iota/iri/network/impl/TipsRequesterImpl.java @@ -6,7 +6,7 @@ import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.Neighbor; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.storage.Tangle; import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; import com.iota.iri.utils.thread.SilentScheduledExecutorService; @@ -29,8 +29,8 @@ public class TipsRequesterImpl implements TipsRequester { private final NeighborRouter neighborRouter; private final Tangle tangle; private final TransactionRequester txRequester; - private final LatestMilestoneTracker latestMilestoneTracker; + private MilestoneSolidifier milestoneSolidifier; private long lastIterationTime = 0; /** @@ -38,16 +38,16 @@ public class TipsRequesterImpl implements TipsRequester { * * @param neighborRouter the {@link NeighborRouter} to use * @param tangle the {@link Tangle} database to load the latest milestone from - * @param latestMilestoneTracker the {@link LatestMilestoneTracker} to gets the latest milestone hash from + * @param milestoneSolidifier the {@link MilestoneSolidifier} to gets the latest milestone hash from * @param txRequester the {@link TransactionRequester} to get the currently number of requested * transactions from */ - public TipsRequesterImpl(NeighborRouter neighborRouter, Tangle tangle, LatestMilestoneTracker latestMilestoneTracker, + public TipsRequesterImpl(NeighborRouter neighborRouter, Tangle tangle, MilestoneSolidifier milestoneSolidifier, TransactionRequester txRequester) { this.neighborRouter = neighborRouter; this.tangle = tangle; - this.latestMilestoneTracker = latestMilestoneTracker; this.txRequester = txRequester; + this.milestoneSolidifier = milestoneSolidifier; } /** @@ -64,7 +64,7 @@ public void start() { public void requestTips() { try { final TransactionViewModel msTVM = TransactionViewModel.fromHash(tangle, - latestMilestoneTracker.getLatestMilestoneHash()); + milestoneSolidifier.getLatestMilestoneHash()); if (msTVM.getBytes().length > 0) { for (Neighbor neighbor : neighborRouter.getConnectedNeighbors().values()) { diff --git a/src/main/java/com/iota/iri/network/pipeline/MilestonePayload.java b/src/main/java/com/iota/iri/network/pipeline/MilestonePayload.java new file mode 100644 index 0000000000..7c63ada07d --- /dev/null +++ b/src/main/java/com/iota/iri/network/pipeline/MilestonePayload.java @@ -0,0 +1,56 @@ +package com.iota.iri.network.pipeline; + +import com.iota.iri.model.Hash; +import com.iota.iri.network.neighbor.Neighbor; + +/** + * A payload object for processing milestone candidates + */ +public class MilestonePayload extends Payload { + /** + * Origin neighbor for transaction + */ + private Neighbor originNeighbor; + + /** + * {@link Hash} of potential milestone object + */ + private Hash milestoneHash; + + /** + * Index of potential milestone object + */ + private int milestoneIndex; + + /** + * Constructor for a {@link MilestonePayload} object that will be processed by the {@link MilestoneStage}. + * + * @param originNeighbor Neighbor that milestone candidate originated from + * @param milestoneHash {@link Hash} of the milestone candidate + * @param milestoneIndex Index of the milestone candidate + */ + public MilestonePayload(Neighbor originNeighbor, Hash milestoneHash, int milestoneIndex){ + this.originNeighbor = originNeighbor; + this.milestoneHash = milestoneHash; + this.milestoneIndex = milestoneIndex; + } + + @Override + public Neighbor getOriginNeighbor() { + return originNeighbor; + } + + /** + * @return {@link #milestoneHash} + */ + public Hash getMilestoneHash(){ + return this.milestoneHash; + } + + /** + * @return {@link #milestoneIndex} + */ + public int getMilestoneIndex(){ + return this.milestoneIndex; + } +} diff --git a/src/main/java/com/iota/iri/network/pipeline/MilestoneStage.java b/src/main/java/com/iota/iri/network/pipeline/MilestoneStage.java new file mode 100644 index 0000000000..a0559cf525 --- /dev/null +++ b/src/main/java/com/iota/iri/network/pipeline/MilestoneStage.java @@ -0,0 +1,107 @@ +package com.iota.iri.network.pipeline; + +import com.iota.iri.controllers.MilestoneViewModel; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.storage.Tangle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Stage for processing {@link MilestonePayload} objects + */ +public class MilestoneStage implements Stage { + + private static final Logger log = LoggerFactory.getLogger(MilestoneStage.class); + + + private MilestoneSolidifier milestoneSolidifier; + private Tangle tangle; + private SnapshotProvider snapshotProvider; + private MilestoneService milestoneService; + private TransactionSolidifier transactionSolidifier; + + /** + * Constructor for {@link MilestoneStage}. This stage will process {@link MilestonePayload} candidate objects using + * the {@link MilestoneSolidifier} to determine the latest milestone, the latest solid milestone, and place unsolid + * milestone candidates into a queue for solidification. The stage will always try to solidify the oldest milestone + * candidate in the queue. + * + * @param tangle The tangle reference + * @param milestoneSolidifier Solidification service for processing milestone objects + * @param snapshotProvider Snapshot provider service for latest snapshot references + * @param milestoneService A service for validating milestone objects + * @param transactionSolidifier A service for solidifying transactions + */ + public MilestoneStage(Tangle tangle, MilestoneSolidifier milestoneSolidifier, + SnapshotProvider snapshotProvider, MilestoneService milestoneService, + TransactionSolidifier transactionSolidifier){ + this.milestoneSolidifier = milestoneSolidifier; + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + this.milestoneService = milestoneService; + this.transactionSolidifier = transactionSolidifier; + } + + /** + * Process {@link MilestonePayload} objects. While processing the {@link MilestoneStage} will determine the latest + * milestone and log it. If the milestone object passes a validity check, it is added to the seenMilestones queue in + * the {@link MilestoneSolidifier}, whereas if it is invalid, the transaction is ignored. The milestone is then + * checked for solidity using the {@link TransactionSolidifier}. If the transaction is solid it is passed forward to + * the {@link TransactionSolidifier} propagation queue. + * + * @param ctx the context to process + * @return Either an abort or solidify stage ctx + */ + @Override + public ProcessingContext process(ProcessingContext ctx){ + try { + MilestonePayload payload = (MilestonePayload) ctx.getPayload(); + + //If the milestone index is below the latest snapshot initial index, then abort the process + if(payload.getMilestoneIndex() < snapshotProvider.getLatestSnapshot().getInitialIndex()){ + return abort(ctx); + } + + TransactionViewModel milestone = TransactionViewModel.fromHash(tangle, payload.getMilestoneHash()); + int newMilestoneIndex = payload.getMilestoneIndex(); + boolean isFirstInBundle = (milestone.getCurrentIndex() == 0); + + // Log new milestones + int latestMilestoneIndex = milestoneSolidifier.getLatestMilestoneIndex(); + if (newMilestoneIndex > latestMilestoneIndex) { + milestoneSolidifier.logNewMilestone(latestMilestoneIndex, newMilestoneIndex, milestone.getHash()); + } + + // Add unsolid milestones to the milestone solidifier, or add solid milestones to the propagation queue + if (!transactionSolidifier.addMilestoneToSolidificationQueue(milestone.getHash(), 50000) && + isFirstInBundle) { + milestoneSolidifier.add(milestone.getHash(), payload.getMilestoneIndex()); + } else { + transactionSolidifier.addToPropagationQueue(milestone.getHash()); + } + + return solidify(ctx, payload, milestone); + }catch (Exception e){ + log.error("Error processing milestone: ", e); + return abort(ctx); + } + } + + + private ProcessingContext solidify(ProcessingContext ctx, Payload payload, TransactionViewModel tvm){ + SolidifyPayload solidifyPayload = new SolidifyPayload(payload.getOriginNeighbor(), tvm); + ctx.setNextStage(TransactionProcessingPipeline.Stage.SOLIDIFY); + ctx.setPayload(solidifyPayload); + return ctx; + } + + private ProcessingContext abort(ProcessingContext ctx){ + ctx.setNextStage(TransactionProcessingPipeline.Stage.ABORT); + return ctx; + } +} diff --git a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java index 6def7d33e8..7cd6f8561f 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ReceivedStage.java @@ -1,5 +1,8 @@ package com.iota.iri.network.pipeline; +import com.iota.iri.model.Hash; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.network.TransactionRequester; @@ -19,27 +22,32 @@ public class ReceivedStage implements Stage { private Tangle tangle; private TransactionRequester transactionRequester; - private TransactionValidator txValidator; + private TransactionSolidifier txSolidifier; private SnapshotProvider snapshotProvider; + private MilestoneService milestoneService; + private Hash cooAddress; /** * Creates a new {@link ReceivedStage}. * * @param tangle The {@link Tangle} database used to store/update the transaction - * @param txValidator The {@link TransactionValidator} used to store/update the transaction + * @param txSolidifier The {@link TransactionValidator} used to store/update the transaction * @param snapshotProvider The {@link SnapshotProvider} used to store/update the transaction */ - public ReceivedStage(Tangle tangle, TransactionValidator txValidator, SnapshotProvider snapshotProvider, - TransactionRequester transactionRequester) { - this.txValidator = txValidator; + public ReceivedStage(Tangle tangle, TransactionSolidifier txSolidifier, SnapshotProvider snapshotProvider, + TransactionRequester transactionRequester, MilestoneService milestoneService, + Hash cooAddress) { + this.txSolidifier = txSolidifier; this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.transactionRequester = transactionRequester; + this.milestoneService = milestoneService; + this.cooAddress = cooAddress; } /** * Stores the given transaction in the database, updates it status - * ({@link TransactionValidator#updateStatus(TransactionViewModel)}) and updates the sender. + * ({@link TransactionSolidifier#updateStatus(TransactionViewModel)}) and updates the sender. * * @param ctx the received stage {@link ProcessingContext} * @return a {@link ProcessingContext} which redirects to the {@link BroadcastStage} @@ -65,7 +73,7 @@ public ProcessingContext process(ProcessingContext ctx) { if (stored) { tvm.setArrivalTime(System.currentTimeMillis()); try { - txValidator.updateStatus(tvm); + txSolidifier.updateStatus(tvm); // free up the recently requested transaction set if(transactionRequester.removeRecentlyRequestedTransaction(tvm.getHash())){ @@ -90,6 +98,19 @@ public ProcessingContext process(ProcessingContext ctx) { transactionRequester.removeRecentlyRequestedTransaction(tvm.getHash()); } + try{ + if(tvm.getAddressHash().equals(cooAddress)) { + int milestoneIndex = milestoneService.getMilestoneIndex(tvm); + MilestonePayload milestonePayload = new MilestonePayload(payload.getOriginNeighbor(), + tvm.getHash(), milestoneIndex); + ctx.setNextStage(TransactionProcessingPipeline.Stage.MILESTONE); + ctx.setPayload(milestonePayload); + return ctx; + } + + } catch(Exception e){ + log.error("Error checking apparent milestone transaction", e); + } // broadcast the newly saved tx to the other neighbors ctx.setNextStage(TransactionProcessingPipeline.Stage.SOLIDIFY); ctx.setPayload(new SolidifyPayload(originNeighbor, tvm)); diff --git a/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java b/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java index 9242e978f4..cb5b7c1d54 100644 --- a/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/ReplyStage.java @@ -10,9 +10,9 @@ import com.iota.iri.network.TransactionCacheDigester; import com.iota.iri.network.neighbor.Neighbor; import com.iota.iri.network.protocol.Protocol; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; +import com.iota.iri.service.milestone.MilestoneSolidifier; import java.security.SecureRandom; @@ -33,7 +33,7 @@ public class ReplyStage implements Stage { private Tangle tangle; private NodeConfig config; private TipsViewModel tipsViewModel; - private LatestMilestoneTracker latestMilestoneTracker; + private MilestoneSolidifier milestoneSolidifier; private SnapshotProvider snapshotProvider; private FIFOCache recentlySeenBytesCache; private SecureRandom rnd = new SecureRandom(); @@ -45,20 +45,20 @@ public class ReplyStage implements Stage { * @param config the {@link NodeConfig} * @param tangle the {@link Tangle} database to load the request transaction from * @param tipsViewModel the {@link TipsViewModel} to load the random tips from - * @param latestMilestoneTracker the {@link LatestMilestoneTracker} to load the latest milestone from + * @param milestoneSolidifier the {@link MilestoneSolidifier} to load the latest milestone from * @param snapshotProvider the {@link SnapshotProvider} to check the latest solid milestone from * @param recentlySeenBytesCache the {@link FIFOCache} to use to cache the replied transaction * @param rnd the {@link SecureRandom} used to get random values to randomize chances for not * replying at all or not requesting a not stored requested transaction from neighbors */ public ReplyStage(NeighborRouter neighborRouter, NodeConfig config, Tangle tangle, TipsViewModel tipsViewModel, - LatestMilestoneTracker latestMilestoneTracker, SnapshotProvider snapshotProvider, - FIFOCache recentlySeenBytesCache, SecureRandom rnd) { + MilestoneSolidifier milestoneSolidifier, SnapshotProvider snapshotProvider, + FIFOCache recentlySeenBytesCache, SecureRandom rnd) { this.neighborRouter = neighborRouter; this.config = config; this.tangle = tangle; this.tipsViewModel = tipsViewModel; - this.latestMilestoneTracker = latestMilestoneTracker; + this.milestoneSolidifier = milestoneSolidifier; this.snapshotProvider = snapshotProvider; this.recentlySeenBytesCache = recentlySeenBytesCache; this.rnd = rnd; @@ -71,18 +71,18 @@ public ReplyStage(NeighborRouter neighborRouter, NodeConfig config, Tangle tangl * @param config the {@link NodeConfig} * @param tangle the {@link Tangle} database to load the request transaction from * @param tipsViewModel the {@link TipsViewModel} to load the random tips from - * @param latestMilestoneTracker the {@link LatestMilestoneTracker} to load the latest milestone from + * @param milestoneSolidifier the {@link MilestoneSolidifier} to load the latest milestone from * @param snapshotProvider the {@link SnapshotProvider} to check the latest solid milestone from * @param recentlySeenBytesCache the {@link FIFOCache} to use to cache the replied transaction */ public ReplyStage(NeighborRouter neighborRouter, NodeConfig config, Tangle tangle, TipsViewModel tipsViewModel, - LatestMilestoneTracker latestMilestoneTracker, SnapshotProvider snapshotProvider, - FIFOCache recentlySeenBytesCache) { + MilestoneSolidifier milestoneSolidifier, SnapshotProvider snapshotProvider, + FIFOCache recentlySeenBytesCache) { this.neighborRouter = neighborRouter; this.config = config; this.tangle = tangle; this.tipsViewModel = tipsViewModel; - this.latestMilestoneTracker = latestMilestoneTracker; + this.milestoneSolidifier = milestoneSolidifier; this.snapshotProvider = snapshotProvider; this.recentlySeenBytesCache = recentlySeenBytesCache; } @@ -106,7 +106,7 @@ public ProcessingContext process(ProcessingContext ctx) { try { // don't reply to random tip requests if we are synchronized with a max delta of one // to the newest milestone - if (snapshotProvider.getLatestSnapshot().getIndex() >= latestMilestoneTracker.getLatestMilestoneIndex() + if (snapshotProvider.getLatestSnapshot().getIndex() >= milestoneSolidifier.getLatestMilestoneIndex() - 1) { ctx.setNextStage(TransactionProcessingPipeline.Stage.FINISH); return ctx; @@ -150,7 +150,7 @@ public ProcessingContext process(ProcessingContext ctx) { // with the latest known milestone and a needed transaction hash, to keep up the ping-pong try { final TransactionViewModel msTVM = TransactionViewModel.fromHash(tangle, - latestMilestoneTracker.getLatestMilestoneHash()); + milestoneSolidifier.getLatestMilestoneHash()); neighborRouter.gossipTransactionTo(neighbor, msTVM, false); } catch (Exception e) { e.printStackTrace(); @@ -161,7 +161,7 @@ public ProcessingContext process(ProcessingContext ctx) { } private Hash getRandomTipPointer() { - Hash tip = rnd.nextDouble() < config.getpSendMilestone() ? latestMilestoneTracker.getLatestMilestoneHash() + Hash tip = rnd.nextDouble() < config.getpSendMilestone() ? milestoneSolidifier.getLatestMilestoneHash() : tipsViewModel.getRandomSolidTipHash(); return tip == null ? Hash.NULL_HASH : tip; } diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java index a56616c96b..6fcadecda3 100644 --- a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java @@ -4,6 +4,9 @@ import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +import com.iota.iri.model.persistables.Transaction; +import com.iota.iri.network.neighbor.Neighbor; +import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Tangle; @@ -28,19 +31,17 @@ public class SolidifyStage implements Stage { private TipsViewModel tipsViewModel; private Tangle tangle; private TransactionViewModel tip; + private MilestoneService milestoneService; /** * Constructor for the {@link SolidifyStage}. * * @param txSolidifier Transaction solidifier implementation for updating transaction's solidity status - * @param txValidator Transaction validator implementation for determining the validity of a transaction * @param tipsViewModel Used for broadcasting random solid tips if the subject transaction is unsolid * @param tangle A reference to the nodes DB */ - public SolidifyStage(TransactionSolidifier txSolidifier, TransactionValidator txValidator, - TipsViewModel tipsViewModel, Tangle tangle){ + public SolidifyStage(TransactionSolidifier txSolidifier, TipsViewModel tipsViewModel, Tangle tangle){ this.txSolidifier = txSolidifier; - this.txValidator = txValidator; this.tipsViewModel = tipsViewModel; this.tangle = tangle; } @@ -61,15 +62,11 @@ public ProcessingContext process(ProcessingContext ctx){ SolidifyPayload payload = (SolidifyPayload) ctx.getPayload(); TransactionViewModel tvm = payload.getTransaction(); - if (tvm.isSolid() || txValidator.quickSetSolid(tvm)) { - ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); - ctx.setPayload(new BroadcastPayload(payload.getOriginNeighbor(), payload.getTransaction())); - return ctx; + if (tvm.isSolid() || txSolidifier.quickSetSolid(tvm)) { + return prepareBroadcast(ctx, payload.getOriginNeighbor(), tvm); } - txSolidifier.addToSolidificationQueue(tvm.getHash()); - - return broadcastTip(ctx, payload); + return broadcastTip(ctx, payload, tvm); }catch (Exception e){ log.error("Failed to process transaction for solidification", e); ctx.setNextStage(TransactionProcessingPipeline.Stage.ABORT); @@ -78,7 +75,8 @@ public ProcessingContext process(ProcessingContext ctx){ } - private ProcessingContext broadcastTip(ProcessingContext ctx, SolidifyPayload payload) throws Exception{ + private ProcessingContext broadcastTip(ProcessingContext ctx, SolidifyPayload payload, + TransactionViewModel tvm) throws Exception{ if(tip == null) { Hash tipHash = tipsViewModel.getRandomSolidTipHash(); @@ -91,12 +89,16 @@ private ProcessingContext broadcastTip(ProcessingContext ctx, SolidifyPayload pa } if(tip.isSolid()) { - ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); - ctx.setPayload(new BroadcastPayload(payload.getOriginNeighbor(), tip)); - return ctx; + return prepareBroadcast(ctx, payload.getOriginNeighbor(), tip); + } else { + return prepareBroadcast(ctx, null, tvm); } + } - ctx.setNextStage(TransactionProcessingPipeline.Stage.FINISH); + private ProcessingContext prepareBroadcast(ProcessingContext ctx, Neighbor originNeighbor, TransactionViewModel tx){ + ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); + ctx.setPayload(new BroadcastPayload(originNeighbor, tx)); + tip = null; return ctx; } diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java index 6a659ef7a9..3b95a31cef 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipeline.java @@ -1,7 +1,6 @@ package com.iota.iri.network.pipeline; import com.iota.iri.network.neighbor.Neighbor; -import com.iota.iri.service.validation.TransactionSolidifier; import java.nio.ByteBuffer; import java.util.concurrent.BlockingQueue; @@ -15,7 +14,7 @@ public interface TransactionProcessingPipeline { * Defines the different stages of the {@link TransactionProcessingPipelineImpl}. */ enum Stage { - PRE_PROCESS, HASHING, VALIDATION, REPLY, RECEIVED, BROADCAST, MULTIPLE, ABORT, FINISH, SOLIDIFY, + PRE_PROCESS, HASHING, VALIDATION, REPLY, RECEIVED, BROADCAST, MULTIPLE, ABORT, FINISH, SOLIDIFY, MILESTONE } /** @@ -51,6 +50,13 @@ enum Stage { */ BlockingQueue getValidationStageQueue(); + /** + * Gets the milestone stage queue. + * + * @return the milestone stage queue + */ + BlockingQueue getMilestoneStageQueue(); + /** * Submits the given data from the given neighbor into the pre processing stage of the pipeline. * @@ -66,12 +72,6 @@ enum Stage { */ void process(byte[] txTrits); - /** - * Fetches a set of transactions from the {@link TransactionSolidifier} and submits - * the object into the {@link BroadcastStage} queue. - */ - void refillBroadcastQueue(); - /** * Shut downs the pipeline by shutting down all stages. */ @@ -125,4 +125,12 @@ enum Stage { * @param solidifyStage the {@link SolidifyStage} to use */ void setSolidifyStage(SolidifyStage solidifyStage); + + /** + * Sets the milestone stage. This method should only be used for injecting mocked objects. + * + * @param milestoneStage the {@link MilestoneStage} to use + */ + void setMilestoneStage(MilestoneStage milestoneStage); + } diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index c401e7beb3..2a60f20488 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -1,5 +1,7 @@ package com.iota.iri.network.pipeline; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.NodeConfig; @@ -15,7 +17,6 @@ import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.TransactionCacheDigester; import com.iota.iri.network.neighbor.Neighbor; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; import com.iota.iri.utils.Converter; @@ -25,6 +26,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; @@ -57,7 +59,7 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingPipeline { private static final Logger log = LoggerFactory.getLogger(TransactionProcessingPipelineImpl.class); - private ExecutorService stagesThreadPool = Executors.newFixedThreadPool(7); + private ExecutorService stagesThreadPool = Executors.newFixedThreadPool(8); // stages of the protocol protocol private PreProcessStage preProcessStage; @@ -68,7 +70,9 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP private BatchedHasher batchedHasher; private HashingStage hashingStage; private SolidifyStage solidifyStage; + private MilestoneStage milestoneStage; private TransactionSolidifier txSolidifier; + private MilestoneSolidifier milestoneSolidifier; private BlockingQueue preProcessStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue validationStageQueue = new ArrayBlockingQueue<>(100); @@ -76,6 +80,7 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP private BlockingQueue replyStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue broadcastStageQueue = new ArrayBlockingQueue<>(100); private BlockingQueue solidifyStageQueue = new ArrayBlockingQueue<>(100); + private BlockingQueue milestoneStageQueue = new ArrayBlockingQueue<>(100); /** * Creates a {@link TransactionProcessingPipeline}. @@ -86,24 +91,29 @@ public class TransactionProcessingPipelineImpl implements TransactionProcessingP * @param tangle The {@link Tangle} database to use to store and load transactions. * @param snapshotProvider The {@link SnapshotProvider} to use to store transactions with. * @param tipsViewModel The {@link TipsViewModel} to load tips from in the reply stage - * @param latestMilestoneTracker The {@link LatestMilestoneTracker} to load the latest milestone hash from in the + * @param milestoneSolidifier The {@link MilestoneSolidifier} to load the latest milestone hash from in the * reply stage */ public TransactionProcessingPipelineImpl(NeighborRouter neighborRouter, NodeConfig config, TransactionValidator txValidator, Tangle tangle, SnapshotProvider snapshotProvider, - TipsViewModel tipsViewModel, LatestMilestoneTracker latestMilestoneTracker, - TransactionRequester transactionRequester, TransactionSolidifier txSolidifier) { + TipsViewModel tipsViewModel, TransactionRequester transactionRequester, + TransactionSolidifier txSolidifier, MilestoneService milestoneService, + MilestoneSolidifier milestoneSolidifier) { FIFOCache recentlySeenBytesCache = new FIFOCache<>(config.getCacheSizeBytes()); this.preProcessStage = new PreProcessStage(recentlySeenBytesCache); - this.replyStage = new ReplyStage(neighborRouter, config, tangle, tipsViewModel, latestMilestoneTracker, + this.replyStage = new ReplyStage(neighborRouter, config, tangle, tipsViewModel, milestoneSolidifier, snapshotProvider, recentlySeenBytesCache); this.broadcastStage = new BroadcastStage(neighborRouter); this.validationStage = new ValidationStage(txValidator, recentlySeenBytesCache); - this.receivedStage = new ReceivedStage(tangle, txValidator, snapshotProvider, transactionRequester); + this.receivedStage = new ReceivedStage(tangle, txSolidifier, snapshotProvider, transactionRequester, + milestoneService, config.getCoordinator()); this.batchedHasher = BatchedHasherFactory.create(BatchedHasherFactory.Type.BCTCURL81, 20); this.hashingStage = new HashingStage(batchedHasher); - this.solidifyStage = new SolidifyStage(txSolidifier, txValidator, tipsViewModel, tangle); + this.solidifyStage = new SolidifyStage(txSolidifier, tipsViewModel, tangle); this.txSolidifier = txSolidifier; + this.milestoneSolidifier = milestoneSolidifier; + this.milestoneStage = new MilestoneStage(tangle, milestoneSolidifier, snapshotProvider, + milestoneService, txSolidifier); } @Override @@ -115,6 +125,7 @@ public void start() { addStage("received", receivedStageQueue, receivedStage); addStage("broadcast", broadcastStageQueue, broadcastStage); addStage("solidify", solidifyStageQueue, solidifyStage); + addStage("milestone", milestoneStageQueue, milestoneStage); } /** @@ -149,6 +160,9 @@ private void addStage(String name, BlockingQueue queue, case BROADCAST: broadcastStageQueue.put(ctx); break; + case MILESTONE: + milestoneStageQueue.put(ctx); + break; case SOLIDIFY: solidifyStageQueue.put(ctx); break; @@ -188,6 +202,11 @@ public BlockingQueue getValidationStageQueue() { return validationStageQueue; } + @Override + public BlockingQueue getMilestoneStageQueue() { + return milestoneStageQueue; + } + @Override public void process(Neighbor neighbor, ByteBuffer data) { try { @@ -207,8 +226,11 @@ public void process(byte[] txTrits) { hashAndValidate(new ProcessingContext(payload)); } - @Override - public void refillBroadcastQueue(){ + /** + * Fetches a set of transactions from the {@link TransactionSolidifier} and submits + * the object into the {@link BroadcastStage} queue. + */ + private void refillBroadcastQueue(){ try{ Iterator hashIterator = txSolidifier.getBroadcastQueue().iterator(); Set toRemove = new LinkedHashSet<>(); @@ -286,4 +308,9 @@ public void setHashingStage(HashingStage hashingStage) { public void setSolidifyStage(SolidifyStage solidifyStage){ this.solidifyStage = solidifyStage; } + + @Override + public void setMilestoneStage(MilestoneStage milestoneStage){ + this.milestoneStage = milestoneStage; + } } diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index a60fa68740..851be14037 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -6,6 +6,8 @@ import com.iota.iri.BundleValidator; import com.iota.iri.IRI; import com.iota.iri.IXI; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.APIConfig; import com.iota.iri.conf.IotaConfig; @@ -21,7 +23,6 @@ import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.dto.*; import com.iota.iri.service.ledger.LedgerService; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.restserver.RestConnector; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; @@ -108,7 +109,8 @@ public class API { private final TipSelector tipsSelector; private final TipsViewModel tipsViewModel; private final TransactionValidator transactionValidator; - private final LatestMilestoneTracker latestMilestoneTracker; + private final MilestoneSolidifier milestoneSolidifier; + private final TransactionSolidifier transactionSolidifier; private final int maxFindTxs; private final int maxRequestList; @@ -148,13 +150,14 @@ public class API { * @param tipsSelector Handles logic for selecting tips based on other transactions * @param tipsViewModel Contains the current tips of this node * @param transactionValidator Validates transactions - * @param latestMilestoneTracker Service that tracks the latest milestone + * @param milestoneSolidifier Service that tracks the latest milestone */ public API(IotaConfig configuration, IXI ixi, TransactionRequester transactionRequester, - SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, - SnapshotProvider snapshotProvider, LedgerService ledgerService, NeighborRouter neighborRouter, TipSelector tipsSelector, - TipsViewModel tipsViewModel, TransactionValidator transactionValidator, - LatestMilestoneTracker latestMilestoneTracker, TransactionProcessingPipeline txPipeline) { + SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, + SnapshotProvider snapshotProvider, LedgerService ledgerService, NeighborRouter neighborRouter, TipSelector tipsSelector, + TipsViewModel tipsViewModel, TransactionValidator transactionValidator, + MilestoneSolidifier milestoneSolidifier, TransactionProcessingPipeline txPipeline, + TransactionSolidifier transactionSolidifier) { this.configuration = configuration; this.ixi = ixi; @@ -169,7 +172,8 @@ public API(IotaConfig configuration, IXI ixi, TransactionRequester transactionRe this.tipsSelector = tipsSelector; this.tipsViewModel = tipsViewModel; this.transactionValidator = transactionValidator; - this.latestMilestoneTracker = latestMilestoneTracker; + this.milestoneSolidifier = milestoneSolidifier; + this.transactionSolidifier = transactionSolidifier; maxFindTxs = configuration.getMaxFindTransactions(); maxRequestList = configuration.getMaxRequestsList(); @@ -425,7 +429,7 @@ private AbstractResponse checkConsistencyStatement(List transactionsList */ private boolean isNodeSynchronized() { return (snapshotProvider.getLatestSnapshot().getIndex() != snapshotProvider.getInitialSnapshot().getIndex()) && - snapshotProvider.getLatestSnapshot().getIndex() >= latestMilestoneTracker.getLatestMilestoneIndex() -1; + snapshotProvider.getLatestSnapshot().getIndex() >= milestoneSolidifier.getLatestMilestoneIndex() -1; } /** @@ -682,7 +686,7 @@ public AbstractResponse storeTransactionsStatement(List trytes) throws E //store transactions if(transactionViewModel.store(tangle, snapshotProvider.getInitialSnapshot())) { transactionViewModel.setArrivalTime(System.currentTimeMillis()); - transactionValidator.updateStatus(transactionViewModel); + transactionSolidifier.updateStatus(transactionViewModel); transactionViewModel.updateSender("local"); transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "sender"); } @@ -721,8 +725,8 @@ private AbstractResponse getNodeInfoStatement() throws Exception{ Runtime.getRuntime().maxMemory(), Runtime.getRuntime().totalMemory(), - latestMilestoneTracker.getLatestMilestoneHash(), - latestMilestoneTracker.getLatestMilestoneIndex(), + milestoneSolidifier.getLatestMilestoneHash(), + milestoneSolidifier.getLatestMilestoneIndex(), snapshotProvider.getLatestSnapshot().getHash(), snapshotProvider.getLatestSnapshot().getIndex(), diff --git a/src/main/java/com/iota/iri/service/milestone/LatestMilestoneTracker.java b/src/main/java/com/iota/iri/service/milestone/LatestMilestoneTracker.java deleted file mode 100644 index 21cc4043bc..0000000000 --- a/src/main/java/com/iota/iri/service/milestone/LatestMilestoneTracker.java +++ /dev/null @@ -1,106 +0,0 @@ -package com.iota.iri.service.milestone; - -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.Hash; - -/** - *

- * The manager that keeps track of the latest milestone by incorporating a background worker that periodically checks if - * new milestones have arrived. - *

- *

- * Knowing about the latest milestone and being able to compare it to the latest solid milestone allows us to determine - * if our node is "in sync". - *

- */ -public interface LatestMilestoneTracker { - /** - *

- * Returns the index of the latest milestone that was seen by this tracker. - *

- *

- * It returns the internal property that is used to store the latest milestone index. - *

- * - * @return the index of the latest milestone that was seen by this tracker - */ - int getLatestMilestoneIndex(); - - /** - *

- * Returns the transaction hash of the latest milestone that was seen by this tracker. - *

- *

- * It returns the internal property that is used to store the latest milestone index. - *

- * @return the transaction hash of the latest milestone that was seen by this tracker - */ - Hash getLatestMilestoneHash(); - - /** - * Sets the latest milestone. - *

- * It stores the passed in values in their corresponding internal properties and can therefore be used to - * inform the {@link LatestSolidMilestoneTracker} about a new milestone. It is internally used to set the new - * milestone but can also be used by tests to mock a certain behaviour or in case we detect a new milestone in other - * parts of the code. - *

- * - * @param latestMilestoneHash the transaction hash of the milestone - * @param latestMilestoneIndex the milestone index of the milestone - */ - void setLatestMilestone(Hash latestMilestoneHash, int latestMilestoneIndex); - - /** - * Analyzes the given transaction to determine if it is a valid milestone. - *

- * If the transaction that was analyzed represents a milestone, we check if it is younger than the current latest - * milestone and update the internal properties accordingly. - *

- * - * @param transaction the transaction that shall be examined - * @return {@code true} if the milestone could be processed and {@code false} if the bundle is not complete, yet - * @throws MilestoneException if anything unexpected happens while trying to analyze the milestone candidate - */ - boolean processMilestoneCandidate(TransactionViewModel transaction) throws MilestoneException; - - /** - * Does the same as {@link #processMilestoneCandidate(TransactionViewModel)} but automatically retrieves the - * transaction belonging to the passed in hash. - * - * @param transactionHash the hash of the transaction that shall be examined - * @return {@code true} if the milestone could be processed and {@code false} if the bundle is not complete, yet - * @throws MilestoneException if anything unexpected happens while trying to analyze the milestone candidate - */ - boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneException; - - /** - *

- * Since the {@link LatestMilestoneTracker} scans all milestone candidates whenever IRI restarts, this flag gives us - * the ability to determine if this initialization process has finished. - *

- *

- * The values returned by {@link #getLatestMilestoneHash()} and {@link #getLatestMilestoneIndex()} will potentially - * return wrong values until the scan has completed. - *

- * - * @return {@code true} if the initial scan of milestones has finished and {@code false} otherwise - */ - boolean isInitialScanComplete(); - - /** - * This method starts the background worker that automatically calls {@link #processMilestoneCandidate(Hash)} on all - * newly found milestone candidates to update the latest milestone. - */ - void start(); - - /** - * This method stops the background worker that updates the latest milestones. - */ - void shutdown(); - - /** - * Initializes the latest milestone tracker by getting the latest milestone. - */ - void init(); -} diff --git a/src/main/java/com/iota/iri/service/milestone/LatestSolidMilestoneTracker.java b/src/main/java/com/iota/iri/service/milestone/LatestSolidMilestoneTracker.java deleted file mode 100644 index ca99037665..0000000000 --- a/src/main/java/com/iota/iri/service/milestone/LatestSolidMilestoneTracker.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.iota.iri.service.milestone; - -import com.iota.iri.service.snapshot.SnapshotProvider; - -/** - *

- * This interface defines the contract for the manager that keeps track of the latest solid milestone by incorporating a - * background worker that periodically checks for new solid milestones. - *

- *

- * Whenever it finds a new solid milestone that hasn't been applied to the ledger state, yet it triggers the application - * logic which in return updates the {@link SnapshotProvider#getLatestSnapshot()}. Since the latest solid milestone is - * encoded in this latest snapshot of the node, this tracker does not introduce separate getters for the latest solid - * milestone. - *

- */ -public interface LatestSolidMilestoneTracker { - /** - *

- * This method searches for new solid milestones that follow the current latest solid milestone and that have not - * been applied to the ledger state yet and applies them. - *

- *

- * It takes care of applying the solid milestones in the correct order by only allowing solid milestones to be - * applied that are directly following our current latest solid milestone. - *

- * - * @throws MilestoneException if anything unexpected happens while updating the latest solid milestone - */ - void trackLatestSolidMilestone() throws MilestoneException; - - /** - * This method starts the background worker that automatically calls {@link #trackLatestSolidMilestone()} - * periodically to keep the latest solid milestone up to date. - */ - void start(); - - /** - * This method stops the background worker that automatically updates the latest solid milestone. - */ - void shutdown(); -} diff --git a/src/main/java/com/iota/iri/service/milestone/MilestoneService.java b/src/main/java/com/iota/iri/service/milestone/MilestoneService.java index 91559d38c8..dc4f495a42 100644 --- a/src/main/java/com/iota/iri/service/milestone/MilestoneService.java +++ b/src/main/java/com/iota/iri/service/milestone/MilestoneService.java @@ -83,7 +83,7 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i *

*

* This allows us to reprocess the milestone in case of errors where the given milestone could not be applied to the - * ledger state. It is for example used by the automatic repair routine of the {@link LatestSolidMilestoneTracker} + * ledger state. It is for example used by the automatic repair routine of the {@link MilestoneSolidifier} * (to recover from inconsistencies due to crashes of IRI). *

*

@@ -104,7 +104,7 @@ MilestoneValidity validateMilestone(TransactionViewModel transactionViewModel, i *

* We determine if the transaction was confirmed by examining its {@code snapshotIndex} value. For this method to * work we require that the previous milestones have been processed already (which is enforced by the {@link - * com.iota.iri.service.milestone.LatestSolidMilestoneTracker} which applies the milestones in the order that they + * com.iota.iri.service.milestone.MilestoneSolidifier} which applies the milestones in the order that they * are issued by the coordinator). *

* diff --git a/src/main/java/com/iota/iri/service/milestone/MilestoneSolidifier.java b/src/main/java/com/iota/iri/service/milestone/MilestoneSolidifier.java index 453c9bee3d..584ce9922a 100644 --- a/src/main/java/com/iota/iri/service/milestone/MilestoneSolidifier.java +++ b/src/main/java/com/iota/iri/service/milestone/MilestoneSolidifier.java @@ -1,7 +1,11 @@ package com.iota.iri.service.milestone; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +import java.util.List; +import java.util.Map; + /** * This interface defines the contract for a manager that tries to solidify unsolid milestones by incorporating a * background worker that periodically checks the solidity of the milestones and issues transaction requests for the @@ -25,4 +29,64 @@ public interface MilestoneSolidifier { * This method shuts down the background worker that asynchronously solidifies the milestones. */ void shutdown(); + + /** + * This method returns the latest milestone index. + * @return Latest Milestone Index + */ + int getLatestMilestoneIndex(); + + /** + * This method returns the latest milestone hash. + * @return Latest Milestone Hash + */ + Hash getLatestMilestoneHash(); + + /** + * Checks if the MilestoneSolidifier has been initialised fully. This registers true if the solidifier has performed + * an initial solidification of present elements in the db. + * + * @return Initialised state. + */ + boolean isInitialScanComplete(); + + /** + * Add to the seen milestones queue to be held for processing solid milestones. In order to be added to the queue + * the transaction must pass {@link MilestoneService#validateMilestone(TransactionViewModel, int)}. + * + * @param milestoneHash The {@link Hash} of the seen milestone + * @param milestoneIndex The index of the seen milestone + */ + void addSeenMilestone(Hash milestoneHash, int milestoneIndex); + + /** + * Removes the specified element from the unsolid milestones queue. In the event that a milestone manages to get + * into the queue, but is later determined to be {@link MilestoneValidity#INVALID}, this method allows it to be + * removed. + * + * @param milestoneHash The {@link Hash} of the milestone to be removed + */ + void removeFromQueue(Hash milestoneHash); + + /** + * Set the latest milestone seen by the node. + * + * @param milestoneHash The latest milestone hash + * @param milestoneIndex The latest milestone index + */ + void setLatestMilestone(Hash milestoneHash, int milestoneIndex); + + /** + * Returns the oldest current milestone object in the unsolid milestone queue to be processed for solidification. + * @return The oldest milestone hash to index mapping + */ + List> getOldestMilestonesInQueue(); + + /** + * Set the latest milestone hash and index, publish and log the update. + * @param oldMilestoneIndex Previous milestone index + * @param newMilestoneIndex New milestone index + * @param newMilestoneHash New milestone hash + */ + void logNewMilestone(int oldMilestoneIndex, int newMilestoneIndex, Hash newMilestoneHash); } diff --git a/src/main/java/com/iota/iri/service/milestone/impl/LatestMilestoneTrackerImpl.java b/src/main/java/com/iota/iri/service/milestone/impl/LatestMilestoneTrackerImpl.java deleted file mode 100644 index 924ef7448f..0000000000 --- a/src/main/java/com/iota/iri/service/milestone/impl/LatestMilestoneTrackerImpl.java +++ /dev/null @@ -1,390 +0,0 @@ -package com.iota.iri.service.milestone.impl; - -import com.iota.iri.conf.IotaConfig; -import com.iota.iri.controllers.AddressViewModel; -import com.iota.iri.controllers.MilestoneViewModel; -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.Hash; -import com.iota.iri.service.milestone.LatestMilestoneTracker; -import com.iota.iri.service.milestone.MilestoneException; -import com.iota.iri.service.milestone.MilestoneService; -import com.iota.iri.service.milestone.MilestoneSolidifier; -import com.iota.iri.service.snapshot.Snapshot; -import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.log.interval.IntervalLogger; -import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; -import com.iota.iri.utils.thread.SilentScheduledExecutorService; - -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -/** - *

- * Creates a tracker that automatically detects new milestones by incorporating a background worker that periodically - * checks all transactions that are originating from the coordinator address and that exposes the found latest milestone - * via getters. - *

- *

- * It can be used to determine the sync-status of the node by comparing these values against the latest solid - * milestone. - *

- */ -public class LatestMilestoneTrackerImpl implements LatestMilestoneTracker { - /** - * Holds the amount of milestone candidates that will be analyzed per iteration of the background worker. - */ - private static final int MAX_CANDIDATES_TO_ANALYZE = 5000; - - /** - * Holds the time (in milliseconds) between iterations of the background worker. - */ - private static final int RESCAN_INTERVAL = 1000; - - /** - * Holds the logger of this class (a rate limited logger than doesn't spam the CLI output). - */ - private static final IntervalLogger log = new IntervalLogger(LatestMilestoneTrackerImpl.class); - - /** - * Holds the Tangle object which acts as a database interface. - */ - private final Tangle tangle; - - /** - * The snapshot provider which gives us access to the relevant snapshots that the node uses (for faster - * bootstrapping). - */ - private final SnapshotProvider snapshotProvider; - - /** - * Service class containing the business logic of the milestone package. - */ - private final MilestoneService milestoneService; - - /** - * Holds a reference to the manager that takes care of solidifying milestones. - */ - private final MilestoneSolidifier milestoneSolidifier; - - /** - * Holds the coordinator address which is used to filter possible milestone candidates. - */ - private final Hash coordinatorAddress; - - /** - * Holds a reference to the manager of the background worker. - */ - private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( - "Latest Milestone Tracker", log.delegate()); - - /** - * Holds the milestone index of the latest milestone that we have seen / processed. - */ - private int latestMilestoneIndex; - - /** - * Holds the transaction hash of the latest milestone that we have seen / processed. - */ - private Hash latestMilestoneHash; - - /** - * A set that allows us to keep track of the candidates that have been seen and added to the {@link - * #milestoneCandidatesToAnalyze} already. - */ - private final Set seenMilestoneCandidates = new HashSet<>(); - - /** - * A list of milestones that still have to be analyzed. - */ - private final Deque milestoneCandidatesToAnalyze = new ArrayDeque<>(); - - /** - * A flag that allows us to detect if the background worker is in its first iteration (for different log - * handling). - */ - private boolean firstRun = true; - - /** - * Flag which indicates if this tracker has finished its initial scan of all old milestone candidates. - */ - private boolean initialized = false; - - /** - * @param tangle Tangle object which acts as a database interface - * @param snapshotProvider manager for the snapshots that allows us to retrieve the relevant snapshots of this node - * @param milestoneService contains the important business logic when dealing with milestones - * @param milestoneSolidifier manager that takes care of solidifying milestones - * @param config configuration object which allows us to determine the important config parameters of the node - */ - public LatestMilestoneTrackerImpl(Tangle tangle, SnapshotProvider snapshotProvider, MilestoneService milestoneService, - MilestoneSolidifier milestoneSolidifier, IotaConfig config) { - this.tangle = tangle; - this.snapshotProvider = snapshotProvider; - this.milestoneService = milestoneService; - this.milestoneSolidifier = milestoneSolidifier; - this.coordinatorAddress = config.getCoordinator(); - } - - @Override - public void init() { - bootstrapLatestMilestoneValue(); - } - - /** - * {@inheritDoc} - * - *

- * In addition to setting the internal properties, we also issue a log message and publish the change to the ZeroMQ - * message processor so external receivers get informed about this change. - *

- */ - @Override - public void setLatestMilestone(Hash latestMilestoneHash, int latestMilestoneIndex) { - tangle.publish("lmi %d %d", this.latestMilestoneIndex, latestMilestoneIndex); - log.delegate().info("Latest milestone has changed from #" + this.latestMilestoneIndex + " to #" + - latestMilestoneIndex); - - this.latestMilestoneHash = latestMilestoneHash; - this.latestMilestoneIndex = latestMilestoneIndex; - } - - @Override - public int getLatestMilestoneIndex() { - return latestMilestoneIndex; - } - - @Override - public Hash getLatestMilestoneHash() { - return latestMilestoneHash; - } - - @Override - public boolean processMilestoneCandidate(Hash transactionHash) throws MilestoneException { - try { - return processMilestoneCandidate(TransactionViewModel.fromHash(tangle, transactionHash)); - } catch (Exception e) { - throw new MilestoneException("unexpected error while analyzing the transaction " + transactionHash, e); - } - } - - /** - * {@inheritDoc} - */ - @Override - public boolean processMilestoneCandidate(TransactionViewModel transaction) throws MilestoneException { - try { - if (coordinatorAddress.equals(transaction.getAddressHash()) && - transaction.getCurrentIndex() == 0) { - - int milestoneIndex = milestoneService.getMilestoneIndex(transaction); - - // if the milestone is older than our ledger start point: we already processed it in the past - if (milestoneIndex <= snapshotProvider.getInitialSnapshot().getIndex()) { - return true; - } - - switch (milestoneService.validateMilestone(transaction, milestoneIndex)) { - case VALID: - if (milestoneIndex > latestMilestoneIndex) { - setLatestMilestone(transaction.getHash(), milestoneIndex); - } - - if (!transaction.isSolid()) { - milestoneSolidifier.add(transaction.getHash(), milestoneIndex); - } - - transaction.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); - break; - - case INCOMPLETE: - return false; - - case INVALID: - // do not re-analyze anymore - return true; - - default: - // we can consider the milestone candidate processed and move on w/o farther action - } - } - - return true; - } catch (Exception e) { - throw new MilestoneException("unexpected error while analyzing the " + transaction, e); - } - } - - @Override - public boolean isInitialScanComplete() { - return initialized; - } - - /** - * {@inheritDoc} - *

- * We repeatedly call {@link #latestMilestoneTrackerThread()} to actively look for new milestones in our database. - * This is a bit inefficient and should at some point maybe be replaced with a check on transaction arrival, but - * this would required adjustments in the whole way IRI handles transactions and is therefore postponed for - * now. - *

- */ - @Override - public void start() { - executorService.silentScheduleWithFixedDelay(this::latestMilestoneTrackerThread, 0, RESCAN_INTERVAL, - TimeUnit.MILLISECONDS); - } - - @Override - public void shutdown() { - executorService.shutdownNow(); - } - - /** - *

- * This method contains the logic for scanning for new latest milestones that gets executed in a background - * worker. - *

- *

- * It first collects all new milestone candidates that need to be analyzed, then analyzes them and finally checks if - * the initialization is complete. In addition to this scanning logic it also issues regular log messages about the - * progress of the scanning. - *

- */ - private void latestMilestoneTrackerThread() { - try { - logProgress(); - collectNewMilestoneCandidates(); - - // additional log message on the first run to indicate how many milestone candidates we have in total - if (firstRun) { - firstRun = false; - - logProgress(); - } - - analyzeMilestoneCandidates(); - checkIfInitializationComplete(); - } catch (MilestoneException e) { - log.error("error while analyzing the milestone candidates", e); - } - } - - /** - *

- * This method emits a log message about the scanning progress. - *

- *

- * It only emits a log message if we have more than one {@link #milestoneCandidatesToAnalyze}, which means that the - * very first call to this method in the "first run" on {@link #latestMilestoneTrackerThread()} will not produce any - * output (which is the reason why we call this method a second time after we have collected all the - * candidates in the "first run"). - *

- */ - private void logProgress() { - if (milestoneCandidatesToAnalyze.size() > 1) { - log.info("Processing milestone candidates (" + milestoneCandidatesToAnalyze.size() + " remaining) ..."); - } - } - - /** - *

- * This method collects the new milestones that have not been "seen" before, by collecting them in the {@link - * #milestoneCandidatesToAnalyze} queue. - *

- *

- * We simply request all transaction that are originating from the coordinator address and treat them as potential - * milestone candidates. - *

- * - * @throws MilestoneException if anything unexpected happens while collecting the new milestone candidates - */ - private void collectNewMilestoneCandidates() throws MilestoneException { - try { - for (Hash hash : AddressViewModel.load(tangle, coordinatorAddress).getHashes()) { - if (Thread.currentThread().isInterrupted()) { - return; - } - - if (seenMilestoneCandidates.add(hash)) { - milestoneCandidatesToAnalyze.addFirst(hash); - } - } - } catch (Exception e) { - throw new MilestoneException("failed to collect the new milestone candidates", e); - } - } - - /** - *

- * This method analyzes the milestone candidates by working through the {@link #milestoneCandidatesToAnalyze} - * queue. - *

- *

- * We only process {@link #MAX_CANDIDATES_TO_ANALYZE} at a time, to give the caller the option to terminate early - * and pick up new milestones as fast as possible without being stuck with analyzing the old ones for too - * long. - *

- * - * @throws MilestoneException if anything unexpected happens while analyzing the milestone candidates - */ - private void analyzeMilestoneCandidates() throws MilestoneException { - int candidatesToAnalyze = Math.min(milestoneCandidatesToAnalyze.size(), MAX_CANDIDATES_TO_ANALYZE); - for (int i = 0; i < candidatesToAnalyze; i++) { - if (Thread.currentThread().isInterrupted()) { - return; - } - - Hash candidateTransactionHash = milestoneCandidatesToAnalyze.pollFirst(); - if(!processMilestoneCandidate(candidateTransactionHash)) { - seenMilestoneCandidates.remove(candidateTransactionHash); - } - } - } - - /** - *

- * This method checks if the initialization is complete. - *

- *

- * It simply checks if the {@link #initialized} flag is not set yet and there are no more {@link - * #milestoneCandidatesToAnalyze}. If the initialization was complete, we issue a log message and set the - * corresponding flag to {@code true}. - *

- */ - private void checkIfInitializationComplete() { - if (!initialized && milestoneCandidatesToAnalyze.size() == 0) { - initialized = true; - - log.info("Processing milestone candidates ... [DONE]").triggerOutput(true); - } - } - - /** - *

- * This method bootstraps this tracker with the latest milestone values that can easily be retrieved without - * analyzing any transactions (for faster startup). - *

- *

- * It first sets the latest milestone to the values found in the latest snapshot and then check if there is a younger - * milestone at the end of our database. While this last entry in the database doesn't necessarily have to be the - * latest one we know it at least gives a reasonable value most of the times. - *

- */ - private void bootstrapLatestMilestoneValue() { - Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); - setLatestMilestone(latestSnapshot.getHash(), latestSnapshot.getIndex()); - - try { - MilestoneViewModel lastMilestoneInDatabase = MilestoneViewModel.latest(tangle); - if (lastMilestoneInDatabase != null && lastMilestoneInDatabase.index() > getLatestMilestoneIndex()) { - setLatestMilestone(lastMilestoneInDatabase.getHash(), lastMilestoneInDatabase.index()); - } - } catch (Exception e) { - log.error("unexpectedly failed to retrieve the latest milestone from the database", e); - } - } -} diff --git a/src/main/java/com/iota/iri/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/com/iota/iri/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java deleted file mode 100644 index 9ae95775a3..0000000000 --- a/src/main/java/com/iota/iri/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ /dev/null @@ -1,471 +0,0 @@ -package com.iota.iri.service.milestone.impl; - -import com.iota.iri.conf.SolidificationConfig; -import com.iota.iri.controllers.MilestoneViewModel; -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.Hash; -import com.iota.iri.network.TransactionRequester; -import com.iota.iri.service.ledger.LedgerService; -import com.iota.iri.service.milestone.LatestMilestoneTracker; -import com.iota.iri.service.milestone.MilestoneException; -import com.iota.iri.service.milestone.MilestoneService; -import com.iota.iri.service.milestone.LatestSolidMilestoneTracker; -import com.iota.iri.service.snapshot.Snapshot; -import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.ASCIIProgressBar; -import com.iota.iri.utils.log.interval.IntervalLogger; -import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; -import com.iota.iri.utils.thread.SilentScheduledExecutorService; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; - -/** - *

- * Creates a manager that keeps track of the latest solid milestones and that triggers the application of these - * milestones and their corresponding balance changes to the latest {@link Snapshot} by incorporating a background - * worker that periodically checks for new solid milestones. - *

- *

- * It extends this with a mechanisms to recover from database corruptions by using a backoff strategy that reverts the - * changes introduced by previous milestones whenever an error is detected until the problem causing milestone was - * found. - *

- */ -public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTracker { - /** - * Holds the interval (in milliseconds) in which the {@link #trackLatestSolidMilestone()} method gets - * called by the background worker. - */ - private static final int RESCAN_INTERVAL = 5000; - - /** - * Holds the logger of this class (a rate limited logger than doesn't spam the CLI output). - */ - private static final IntervalLogger log = new IntervalLogger(LatestSolidMilestoneTrackerImpl.class); - - /** - * Holds the Tangle object which acts as a database interface. - */ - private final Tangle tangle; - - /** - * The snapshot provider which gives us access to the relevant snapshots that the node uses (for the ledger - * state). - */ - private final SnapshotProvider snapshotProvider; - - /** - * Holds a reference to the service instance containing the business logic of the milestone package. - */ - private final MilestoneService milestoneService; - - /** - * Holds a reference to the manager that keeps track of the latest milestone. - */ - private final LatestMilestoneTracker latestMilestoneTracker; - - /** - * Holds a reference to the service that contains the logic for applying milestones to the ledger state. - */ - private final LedgerService ledgerService; - - /** - * Holds a reference to the manager of the background worker. - */ - private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( - "Latest Solid Milestone Tracker", log.delegate()); - - /** - * Boolean flag that is used to identify the first iteration of the background worker. - */ - private boolean firstRun = true; - - /** - * Holds the milestone index of the milestone that caused the repair logic to get started. - */ - private int errorCausingMilestoneIndex = Integer.MAX_VALUE; - - /** - * Counter for the backoff repair strategy (see {@link #repairCorruptedMilestone(MilestoneViewModel)}. - */ - private int repairBackoffCounter = 0; - - /** - * Uses to clear the request queue once we are fully synced. - */ - private TransactionRequester transactionRequester; - - /** - * Holds information about the current synchronisation progress. - */ - private SyncProgressInfo syncProgressInfo = new SyncProgressInfo(); - - private SolidificationConfig solidificationConfig; - - /** - *

- * This method initializes the instance and registers its dependencies. - *

- *

- * It stores the passed in values in their corresponding private properties. - *

- *

- * Note: Instead of handing over the dependencies in the constructor, we register them lazy. This allows us to have - * circular dependencies because the instantiation is separated from the dependency injection. To reduce the - * amount of code that is necessary to correctly instantiate this class, we return the instance itself which - * allows us to still instantiate, initialize and assign in one line - see Example: - *

- * {@code latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl().init(...);} - * - * @param tangle Tangle object which acts as a database interface - * @param snapshotProvider manager for the snapshots that allows us to retrieve the relevant snapshots of this node - * @param milestoneService contains the important business logic when dealing with milestones - * @param ledgerService the manager for - * @param latestMilestoneTracker the manager that keeps track of the latest milestone - * @param transactionRequester the manager which keeps and tracks transactions which are requested - * @return the initialized instance itself to allow chaining - */ - public LatestSolidMilestoneTrackerImpl(Tangle tangle, SnapshotProvider snapshotProvider, - MilestoneService milestoneService, LedgerService ledgerService, - LatestMilestoneTracker latestMilestoneTracker, TransactionRequester transactionRequester, - SolidificationConfig solidificationConfig) { - - this.tangle = tangle; - this.snapshotProvider = snapshotProvider; - this.milestoneService = milestoneService; - this.ledgerService = ledgerService; - this.latestMilestoneTracker = latestMilestoneTracker; - this.transactionRequester = transactionRequester; - this.solidificationConfig = solidificationConfig; - } - - @Override - public void start() { - executorService.silentScheduleWithFixedDelay(this::latestSolidMilestoneTrackerThread, 0, RESCAN_INTERVAL, - TimeUnit.MILLISECONDS); - } - - @Override - public void shutdown() { - executorService.shutdownNow(); - } - - /** - * {@inheritDoc} - * - *

- * In addition to applying the found milestones to the ledger state it also issues log messages and keeps the - * {@link LatestMilestoneTracker} in sync (if we happen to process a new latest milestone faster). - *

- */ - @Override - public void trackLatestSolidMilestone() throws MilestoneException { - try { - int currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); - if (currentSolidMilestoneIndex < latestMilestoneTracker.getLatestMilestoneIndex()) { - MilestoneViewModel nextMilestone; - while (!Thread.currentThread().isInterrupted() && - (nextMilestone = MilestoneViewModel.get(tangle, currentSolidMilestoneIndex + 1)) != null && - TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) { - - syncLatestMilestoneTracker(nextMilestone.getHash(), nextMilestone.index()); - applySolidMilestoneToLedger(nextMilestone); - logChange(currentSolidMilestoneIndex); - - currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); - if (currentSolidMilestoneIndex == latestMilestoneTracker.getLatestMilestoneIndex()) { - transactionRequester.clearRecentlyRequestedTransactions(); - } - } - } else { - syncLatestMilestoneTracker(snapshotProvider.getLatestSnapshot().getHash(), - currentSolidMilestoneIndex); - } - } catch (Exception e) { - throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); - } - } - - /** - *

- * Contains the logic for the background worker. - *

- *

- * It simply calls {@link #trackLatestSolidMilestone()} and wraps with a log handler that prevents the {@link - * MilestoneException} to crash the worker. - *

- */ - private void latestSolidMilestoneTrackerThread() { - try { - if (firstRun) { - firstRun = false; - - ledgerService.restoreLedgerState(); - int milestoneStartIndex = snapshotProvider.getInitialSnapshot().getIndex(); - syncProgressInfo.setSyncMilestoneStartIndex(milestoneStartIndex); - logChange(milestoneStartIndex); - } - - trackLatestSolidMilestone(); - } catch (Exception e) { - log.error("error while updating the solid milestone", e); - } - } - - /** - *

- * Applies the given milestone to the ledger. - *

- *

- * If the application of the milestone fails, we start a repair routine which will revert the milestones preceding - * our current milestone and consequently try to reapply them in the next iteration of the {@link - * #trackLatestSolidMilestone()} method (until the problem is solved). - *

- * - * @param milestone the milestone that shall be applied to the ledger state - * @throws Exception if anything unexpected goes wrong while applying the milestone to the ledger - */ - private void applySolidMilestoneToLedger(MilestoneViewModel milestone) throws Exception { - if (ledgerService.applyMilestoneToLedger(milestone)) { - if (isRepairRunning() && isRepairSuccessful(milestone)) { - stopRepair(); - } - syncProgressInfo.addMilestoneApplicationTime(); - } else { - repairCorruptedMilestone(milestone); - } - } - - /** - *

- * Checks if we are currently trying to repair a milestone. - *

- *

- * We simply use the {@link #repairBackoffCounter} as an indicator if a repair routine is running. - *

- * - * @return {@code true} if we are trying to repair a milestone and {@code false} otherwise - */ - private boolean isRepairRunning() { - return repairBackoffCounter != 0; - } - - /** - *

- * Checks if we successfully repaired the corrupted milestone. - *

- *

- * To determine if the repair routine was successful we check if the processed milestone has a higher index than the - * one that initially could not get applied to the ledger. - *

- * - * @param processedMilestone the currently processed milestone - * @return {@code true} if we advanced to a milestone following the corrupted one and {@code false} otherwise - */ - private boolean isRepairSuccessful(MilestoneViewModel processedMilestone) { - return processedMilestone.index() > errorCausingMilestoneIndex; - } - - /** - *

- * Resets the internal variables that are used to keep track of the repair process. - *

- *

- * It gets called whenever we advance to a milestone that has a higher milestone index than the milestone that - * initially caused the repair routine to kick in (see {@link #repairCorruptedMilestone(MilestoneViewModel)}. - *

- */ - private void stopRepair() { - repairBackoffCounter = 0; - errorCausingMilestoneIndex = Integer.MAX_VALUE; - } - - /** - *

- * Keeps the {@link LatestMilestoneTracker} in sync with our current progress. - *

- *

- * Since the {@link LatestMilestoneTracker} scans all old milestones during its startup (which can take a while to - * finish) it can happen that we see a newer latest milestone faster than this manager. - *

- * Note: This method ensures that the latest milestone index is always bigger or equals the latest solid milestone - * index. - * - * @param milestoneHash transaction hash of the milestone - * @param milestoneIndex milestone index - */ - private void syncLatestMilestoneTracker(Hash milestoneHash, int milestoneIndex) { - if (milestoneIndex > latestMilestoneTracker.getLatestMilestoneIndex()) { - latestMilestoneTracker.setLatestMilestone(milestoneHash, milestoneIndex); - } - } - - /** - *

- * Emits a log message whenever the latest solid milestone changes. - *

- *

- * It simply compares the current latest milestone index against the previous milestone index and emits the log - * messages using the {@link #log} and the {@link #messageQ} instances if it differs. - *

- * - * @param prevSolidMilestoneIndex the milestone index before the change - */ - private void logChange(int prevSolidMilestoneIndex) { - Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); - int latestSolidMilestoneIndex = latestSnapshot.getIndex(); - - if (prevSolidMilestoneIndex == latestSolidMilestoneIndex) { - return; - } - - Hash latestMilestoneHash = latestSnapshot.getHash(); - log.info("Latest SOLID milestone index changed from #" + prevSolidMilestoneIndex + " to #" + latestSolidMilestoneIndex); - - tangle.publish("lmsi %d %d", prevSolidMilestoneIndex, latestSolidMilestoneIndex); - tangle.publish("lmhs %s", latestMilestoneHash); - - if (!solidificationConfig.isPrintSyncProgressEnabled()) { - return; - } - - int latestMilestoneIndex = latestMilestoneTracker.getLatestMilestoneIndex(); - - // only print more sophisticated progress if we are coming from a more unsynced state - if(latestMilestoneIndex - latestSolidMilestoneIndex < 1) { - syncProgressInfo.setSyncMilestoneStartIndex(latestSolidMilestoneIndex); - syncProgressInfo.resetMilestoneApplicationTimes(); - return; - } - - int estSecondsToBeSynced = syncProgressInfo.computeEstimatedTimeToSyncUpSeconds(latestMilestoneIndex, - latestSolidMilestoneIndex); - StringBuilder progressSB = new StringBuilder(); - - // add progress bar - progressSB.append(ASCIIProgressBar.getProgressBarString(syncProgressInfo.getSyncMilestoneStartIndex(), - latestMilestoneIndex, latestSolidMilestoneIndex)); - // add lsm to lm - progressSB.append(String.format(" [LSM %d / LM %d - remaining: %d]", latestSolidMilestoneIndex, - latestMilestoneIndex, latestMilestoneIndex - latestSolidMilestoneIndex)); - // add estimated time to get fully synced - if (estSecondsToBeSynced != -1) { - progressSB.append(String.format(" - est. seconds to get synced: %d", estSecondsToBeSynced)); - } - log.info(progressSB.toString()); - } - - - /** - *

- * Tries to actively repair the ledger by reverting the milestones preceding the given milestone. - *

- *

- * It gets called when a milestone could not be applied to the ledger state because of problems like "inconsistent - * balances". While this should theoretically never happen (because milestones are by definition "consistent"), it - * can still happen because IRI crashed or got stopped in the middle of applying a milestone or if a milestone - * was processed in the wrong order. - *

- *

- * Every time we call this method the internal {@link #repairBackoffCounter} is incremented which causes the next - * call of this method to repair an additional milestone. This means that whenever we face an error we first try to - * reset only the last milestone, then the two last milestones, then the three last milestones (and so on ...) until - * the problem was fixed. - *

- *

- * To be able to tell when the problem is fixed and the {@link #repairBackoffCounter} can be reset, we store the - * milestone index that caused the problem the first time we call this method. - *

- * - * @param errorCausingMilestone the milestone that failed to be applied - * @throws MilestoneException if we failed to reset the corrupted milestone - */ - private void repairCorruptedMilestone(MilestoneViewModel errorCausingMilestone) throws MilestoneException { - if (repairBackoffCounter++ == 0) { - errorCausingMilestoneIndex = errorCausingMilestone.index(); - } - for (int i = errorCausingMilestone.index(); i > errorCausingMilestone.index() - repairBackoffCounter; i--) { - milestoneService.resetCorruptedMilestone(i); - } - } - - /** - * Holds variables containing information needed for sync progress calculation. - */ - private static class SyncProgressInfo { - /** - * The actual start milestone index from which the node started from when syncing up. - */ - private int syncMilestoneStartIndex; - - /** - * Used to calculate the average time needed to apply a milestone. - */ - private BlockingQueue lastMilestoneApplyTimes = new ArrayBlockingQueue<>(100); - - /** - * Gets the milestone sync start index. - * - * @return the milestone sync start index - */ - int getSyncMilestoneStartIndex() { - return syncMilestoneStartIndex; - } - - /** - * Sets the milestone sync start index. - * - * @param syncMilestoneStartIndex the value to set - */ - void setSyncMilestoneStartIndex(int syncMilestoneStartIndex) { - this.syncMilestoneStartIndex = syncMilestoneStartIndex; - } - - /** - * Adds the current time as a time where a milestone got applied to the ledger state. - */ - void addMilestoneApplicationTime(){ - if(lastMilestoneApplyTimes.remainingCapacity() == 0){ - lastMilestoneApplyTimes.remove(); - } - lastMilestoneApplyTimes.add(System.currentTimeMillis()); - } - - /** - * Clears the records of times when milestones got applied. - */ - void resetMilestoneApplicationTimes(){ - lastMilestoneApplyTimes.clear(); - } - - /** - * Computes the estimated time (seconds) needed to get synced up by looking at the last N milestone applications - * times and the current solid and last milestone indices. - * - * @param latestMilestoneIndex the current latest known milestone index - * @param latestSolidMilestoneIndex the current to the ledger applied milestone index - * @return the number of seconds needed to get synced up or -1 if not enough data is available - */ - int computeEstimatedTimeToSyncUpSeconds(int latestMilestoneIndex, int latestSolidMilestoneIndex) { - // compute average time needed to apply a milestone - Object[] times = lastMilestoneApplyTimes.toArray(); - long sumDelta = 0; - double avgMilestoneApplyMillisec; - if (times.length > 1) { - // compute delta sum - for (int i = times.length - 1; i > 1; i--) { - sumDelta += ((Long)times[i]) - ((Long)times[i - 1]); - } - avgMilestoneApplyMillisec = (double) sumDelta / (double) (times.length - 1); - } else { - return -1; - } - - return (int) ((avgMilestoneApplyMillisec / 1000) * (latestMilestoneIndex - latestSolidMilestoneIndex)); - } - } - -} diff --git a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java index 6a2aa6fc40..74a8236472 100644 --- a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java @@ -1,351 +1,666 @@ package com.iota.iri.service.milestone.impl; -import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.conf.SolidificationConfig; +import com.iota.iri.controllers.AddressViewModel; +import com.iota.iri.controllers.MilestoneViewModel; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.ledger.LedgerService; +import com.iota.iri.service.milestone.MilestoneException; +import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.milestone.MilestoneValidity; +import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.validation.TransactionSolidifier; +import com.iota.iri.storage.Tangle; +import com.iota.iri.utils.ASCIIProgressBar; import com.iota.iri.utils.log.interval.IntervalLogger; import com.iota.iri.utils.thread.DedicatedScheduledExecutorService; import com.iota.iri.utils.thread.SilentScheduledExecutorService; import java.util.*; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -/** - *

- * This class implements the basic contract of the {@link MilestoneSolidifier} interface. - *

- *

- * It manages a map of unsolid milestones to collect all milestones that have to be solidified. It then periodically - * issues checkSolidity calls on the earliest milestones to solidify them. - *

- *

- * To save resources and make the call a little bit more efficient, we cache the earliest milestones in a separate map, - * so the relatively expensive task of having to search for the next earliest milestone in the pool only has to be - * performed after a milestone has become solid or irrelevant for our node. - *

- */ +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ArrayBlockingQueue; + public class MilestoneSolidifierImpl implements MilestoneSolidifier { - /** - * Defines the amount of milestones that we "simultaneously" try to solidify in one pass. - */ - private static final int SOLIDIFICATION_QUEUE_SIZE = 20; + private static final IntervalLogger log = new IntervalLogger(MilestoneSolidifierImpl.class); + // Max size fo the solidification queue + private static final int MAX_SIZE = 10; - /** - * Defines the interval in which solidity checks are issued (in milliseconds). - */ - private static final int SOLIDIFICATION_INTERVAL = 500; + private Map unsolidMilestones = new ConcurrentHashMap<>(); + private Map solidificationQueue = new ConcurrentHashMap<>(); + private Map seenMilestones = new ConcurrentHashMap<>(); - /** - *

- * Defines the maximum amount of transactions that are allowed to get processed while trying to solidify a - * milestone. - *

- *

- * Note: We want to find the next previous milestone and not get stuck somewhere at the end of the tangle with a - * long running {@link TransactionSolidifier#checkSolidity(Hash)} call. - *

- */ - private static final int SOLIDIFICATION_TRANSACTIONS_LIMIT = 50000; + private Map.Entry oldestMilestoneInQueue = null; + + private int latestMilestoneIndex; + private Hash latestMilestoneHash; + private int latestSolidMilestone; + + private TransactionSolidifier transactionSolidifier; + private LedgerService ledgerService; + private SnapshotProvider snapshotProvider; + private Tangle tangle; + private TransactionRequester transactionRequester; + private MilestoneService milestoneService; + private SolidificationConfig config; + + + private SyncProgressInfo syncProgressInfo = new SyncProgressInfo(); /** - * Logger for this class allowing us to dump debug and status messages. + * Holds the milestone index of the milestone that caused the repair logic to get started. */ - private static final IntervalLogger log = new IntervalLogger(MilestoneSolidifier.class); + private int errorCausingMilestoneIndex = Integer.MAX_VALUE; /** - * Holds the snapshot provider which gives us access to the relevant snapshots. + * Counter for the backoff repair strategy (see {@link #repairCorruptedMilestone(MilestoneViewModel)}. */ - private final SnapshotProvider snapshotProvider; + private int repairBackoffCounter = 0; /** - * Holds a reference to the transactionSolidifier which allows us to issue solidity checks. + * An indicator for whether or not the solidifier has processed from the first batch of seen milestones */ - private final TransactionSolidifier transactionSolidifier; + private boolean initialized = false; /** - * Holds a reference to the manager of the background worker. + * The execution service for the milestone solidifier thread */ private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService( - "Milestone Solidifier", log.delegate()); + "Milestone Solidifier"); + + private boolean firstRun = true; + /** - *

- * Holds the milestones that were newly added, but not examined yet. - *

- *

- * Note: This is used to be able to add milestones to the solidifier without having to synchronize the access to the - * underlying Maps. - *

+ * Constructor for the {@link MilestoneSolidifierImpl}. This class holds milestone objects to be processed for + * solidification. It also tracks the latest solid milestone object. + * + * @param transactionSolidifier Solidification service for transaction objects + * @param tangle DB reference + * @param snapshotProvider SnapshotProvider for fetching local snapshots + * @param ledgerService LedgerService for bootstrapping ledger state + * @param txRequester TransationRequester to submit missing transactions to + * @param milestoneService MilestoneService for resetting corrupted milestones + * @param config Configuration to determine if additional logging will be used */ - private final Map newlyAddedMilestones = new ConcurrentHashMap<>(); + public MilestoneSolidifierImpl(TransactionSolidifier transactionSolidifier, Tangle tangle, + SnapshotProvider snapshotProvider, LedgerService ledgerService, + TransactionRequester txRequester, MilestoneService milestoneService, + SolidificationConfig config){ + this.transactionSolidifier = transactionSolidifier; + this.snapshotProvider = snapshotProvider; + this.ledgerService = ledgerService; + this.tangle = tangle; + this.transactionRequester = txRequester; + this.milestoneService = milestoneService; + this.config = config; + } + + @Override + public void start(){ + executorService.silentExecute(this::milestoneSolidificationThread); + } + + @Override + public void shutdown(){ + executorService.shutdownNow(); + } + + + private void milestoneSolidificationThread() { + while(!Thread.currentThread().isInterrupted()) { + try { + if (firstRun) { + firstRun = false; + latestMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex(); + latestMilestoneHash = snapshotProvider.getLatestSnapshot().getHash(); + + bootStrapSolidMilestones(); + ledgerService.restoreLedgerState(); + logChange(snapshotProvider.getInitialSnapshot().getIndex()); + + syncProgressInfo.setSyncMilestoneStartIndex(snapshotProvider.getInitialSnapshot().getIndex()); + } + solidifySeenMilestones(); + processSolidifyQueue(); + + checkLatestSolidMilestone(); + solidifyLog(); + } catch(Exception e){ + log.error("Error running milestone solidification thread", e); + } + } + } + + private void solidifyLog(){ + if(unsolidMilestones.size() > 1){ + if(oldestMilestoneInQueue != null){ + int milestoneIndex = oldestMilestoneInQueue.getValue(); + log.info("Solidifying milestone # " + milestoneIndex + " - [ LSM: " + latestSolidMilestone + + " LM: " + latestMilestoneIndex + " ] - [ Remaining: " + + (latestMilestoneIndex - latestSolidMilestone) + " Queued: " + + (seenMilestones.size() + unsolidMilestones.size()) + " ]"); + } + } + } /** - * Holds all unsolid milestones that shall be solidified (the transaction hash mapped to its milestone index). + * Iterates through seen milestones and ensures that they are valid. If the milestone is invalid, the index is + * checked to see if a milestone object exists already. If an existing milestone is present, then the current + * milestone index/hash pairing is replaced with the existing milestone hash. */ - private final Map unsolidMilestonesPool = new ConcurrentHashMap<>(); + private void solidifySeenMilestones() throws Exception{ + Iterator> iterator = seenMilestones.entrySet().iterator(); + Map.Entry milestone; + while(!Thread.currentThread().isInterrupted() && iterator.hasNext()){ + milestone = iterator.next(); + int milestoneIndex = milestone.getKey(); + TransactionViewModel oldMilestoneTransaction = TransactionViewModel.fromHash(tangle, milestone.getValue()); + if(milestoneService.validateMilestone(oldMilestoneTransaction, milestoneIndex) + .equals(MilestoneValidity.INVALID)){ + MilestoneViewModel nextMilestone = MilestoneViewModel.get(tangle, milestoneIndex); + if(nextMilestone != null) { + if(TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) { + seenMilestones.remove(milestoneIndex); + removeFromQueue(oldMilestoneTransaction.getHash()); + add(nextMilestone.getHash(), milestoneIndex); + } + } else { + seenMilestones.remove(milestoneIndex); + removeFromQueue(oldMilestoneTransaction.getHash()); + } + } + } + } /** - * Holds the milestones that are actively trying to be solidified by the background {@link Thread} (acts as a - * Queue). + * Iterates through valid milestones and submits them to the {@link TransactionSolidifier} */ - private final Map milestonesToSolidify = new HashMap<>(); + private void processSolidifyQueue() throws Exception { + Iterator> iterator = solidificationQueue.entrySet().iterator(); + Map.Entry milestone; + while(!Thread.currentThread().isInterrupted() && iterator.hasNext()){ + milestone = iterator.next(); + TransactionViewModel milestoneCandidate = TransactionViewModel.fromHash(tangle, milestone.getKey()); + if(milestoneService.validateMilestone(milestoneCandidate, milestone.getValue()) + .equals(MilestoneValidity.VALID)){ + milestoneCandidate.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); + if(milestoneCandidate.isSolid()) { + addSeenMilestone(milestone.getKey(), milestone.getValue()); + } else { + transactionSolidifier.addToSolidificationQueue(milestone.getKey()); + } + } else { + transactionSolidifier.addToSolidificationQueue(milestone.getKey()); + } + } + } + /** - *

- * Holds and entry that represents the youngest milestone in the {@link #milestonesToSolidify} Map. - *

- *

- * Note: It is used to check if new milestones that are being added, are older that the currently processed ones and - * should replace them in the queue (we solidify from oldest to youngest). - *

+ * Iterates through milestone indexes starting from the initial snapshot index, and returns the newest existing + * milestone index + * + * @return Latest Solid Milestone index + * @throws Exception */ - private Map.Entry youngestMilestoneInQueue = null; + private void bootStrapSolidMilestones() throws Exception { + boolean solid = true; + int milestoneIndex = snapshotProvider.getInitialSnapshot().getIndex(); + while(solid){ + if(!MilestoneViewModel.load(tangle, milestoneIndex)){ + solid = false; + } + milestoneIndex += 1; + } + latestSolidMilestone = milestoneIndex; + + AddressViewModel.load(tangle, config.getCoordinator()).getHashes().forEach(hash -> { + try { + int index; + if ((index = milestoneService.getMilestoneIndex(TransactionViewModel.fromHash(tangle, hash))) > + latestSolidMilestone){ + add(hash, index); + latestMilestoneIndex = index; + latestMilestoneHash = hash; + + } + } catch(Exception e) { + log.error("Error processing existing milestone index", e); + } + }); + + if(latestMilestoneIndex > snapshotProvider.getInitialSnapshot().getIndex()) { + logNewMilestone(snapshotProvider.getInitialSnapshot().getIndex(), latestMilestoneIndex, latestMilestoneHash); + } + } + /** - * @param snapshotProvider snapshot provider which gives us access to the relevant snapshots - * @param transactionSolidifier transactionSolidifier instance that is used by the node + * Tries to solidifiy the next milestone index. If successful, the milestone will be removed from the + * {@link #seenMilestones} queue, and any milestone objects below that index in the {@link #unsolidMilestones} queue + * will be removed as well. */ - public MilestoneSolidifierImpl(SnapshotProvider snapshotProvider, TransactionSolidifier transactionSolidifier) { - this.snapshotProvider = snapshotProvider; - this.transactionSolidifier = transactionSolidifier; + private void checkLatestSolidMilestone(){ + try { + if (latestMilestoneIndex > latestSolidMilestone) { + int nextMilestone = latestSolidMilestone + 1; + if (seenMilestones.containsKey(nextMilestone)){ + TransactionViewModel milestone = TransactionViewModel.fromHash(tangle, + seenMilestones.get(nextMilestone)); + if(milestone.isSolid()){ + updateSolidMilestone(latestSolidMilestone); + transactionSolidifier.addToPropagationQueue(milestone.getHash()); + } else { + transactionSolidifier.addToSolidificationQueue(milestone.getHash()); + } + } + } + } catch (Exception e) { + log.info(e.getMessage()); + } + } + + private void checkValidity(TransactionViewModel milestone, int index) throws Exception{ + switch(milestoneService.validateMilestone(milestone, index)) { + case INVALID: + MilestoneViewModel nextMilestone = MilestoneViewModel.get(tangle, index); + if(nextMilestone != null) { + if(TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) { + seenMilestones.remove(index); + removeFromQueue(milestone.getHash()); + add(nextMilestone.getHash(), index); + } + } else { + removeFromQueue(milestone.getHash()); + } + case INCOMPLETE: + transactionSolidifier.addToSolidificationQueue(milestone.getHash()); + } + } + + private void updateSolidMilestone(int currentSolidMilestoneIndex) throws Exception{ + MilestoneViewModel nextSolidMilestone = MilestoneViewModel.get(tangle, currentSolidMilestoneIndex + 1); + if(nextSolidMilestone != null) { + applySolidMilestoneToLedger(nextSolidMilestone); + logChange(currentSolidMilestoneIndex); + + if (currentSolidMilestoneIndex + 1 == latestMilestoneIndex) { + transactionRequester.clearRecentlyRequestedTransactions(); + } + + removeSeenMilestone(currentSolidMilestoneIndex + 1); + latestSolidMilestone = currentSolidMilestoneIndex + 1; + } + } /** * {@inheritDoc} - * - *

- * Since this method might be called from a performance critical context, we simply add the milestone to a temporary - * pool, that gets examined later by the background process. This doesn't just speed up the addition of new jobs but - * also prevents us from having to synchronize the access to the underlying maps. - *

*/ @Override - public void add(Hash milestoneHash, int milestoneIndex) { - if (!unsolidMilestonesPool.containsKey(milestoneHash) && !newlyAddedMilestones.containsKey(milestoneHash) && - milestoneIndex > snapshotProvider.getInitialSnapshot().getIndex()) { + public void add(Hash milestoneHash, int milestoneIndex){ + if(!unsolidMilestones.containsKey(milestoneHash) && !seenMilestones.containsKey(milestoneIndex) && + milestoneIndex > latestSolidMilestone){ + unsolidMilestones.put(milestoneHash, milestoneIndex); + updateQueues(milestoneHash, milestoneIndex); + } + } - newlyAddedMilestones.put(milestoneHash, milestoneIndex); + private void updateQueues(Hash milestoneHash, int milestoneIndex){ + if(solidificationQueue.containsKey(milestoneHash)) { + if(solidificationQueue.size() >= MAX_SIZE){ + Iterator> iterator = solidificationQueue.entrySet().iterator(); + iterator.next(); + iterator.remove(); + } } + + solidificationQueue.put(milestoneHash, milestoneIndex); + scanMilestonesInQueue(); } + /** + * {@inheritDoc} + */ @Override - public void start() { - executorService.silentScheduleWithFixedDelay(this::milestoneSolidificationThread, 0, SOLIDIFICATION_INTERVAL, - TimeUnit.MILLISECONDS); + public void addSeenMilestone(Hash milestoneHash, int milestoneIndex){ + if(!seenMilestones.containsKey(milestoneIndex)){ + seenMilestones.put(milestoneIndex, milestoneHash); + } + removeFromQueue(milestoneHash); } + /** + * {@inheritDoc} + */ @Override - public void shutdown() { - executorService.shutdownNow(); + public List >getOldestMilestonesInQueue(){ + return new ArrayList<>(solidificationQueue.entrySet()); } /** - *

- * This method takes an entry from the {@link #unsolidMilestonesPool} and adds it to the - * {@link #milestonesToSolidify} queue. - *

- *

- * It first checks if the given milestone is already part of the queue and then tries to add it. If the queue is not - * full yet, the addition to the queue is relatively cheap, because the {@link #youngestMilestoneInQueue} marker can - * be updated without iterating over all entries. If the queue reached its capacity already, we replace the entry - * marked by the {@link #youngestMilestoneInQueue} marker and update the marker by recalculating it using - * {@link #determineYoungestMilestoneInQueue()}. - *

- * - * @param milestoneEntry entry from the {@link #unsolidMilestonesPool} that shall get added to the queue + * {@inheritDoc} */ - private void addToSolidificationQueue(Map.Entry milestoneEntry) { - if (milestonesToSolidify.containsKey(milestoneEntry.getKey())) { - return; - } - - if (milestonesToSolidify.size() < SOLIDIFICATION_QUEUE_SIZE) { - milestonesToSolidify.put(milestoneEntry.getKey(), milestoneEntry.getValue()); - - if (youngestMilestoneInQueue == null || milestoneEntry.getValue() > youngestMilestoneInQueue.getValue()) { - youngestMilestoneInQueue = milestoneEntry; - } - } else if (milestoneEntry.getValue() < youngestMilestoneInQueue.getValue()) { - milestonesToSolidify.remove(youngestMilestoneInQueue.getKey()); - milestonesToSolidify.put(milestoneEntry.getKey(), milestoneEntry.getValue()); + @Override + public void removeFromQueue(Hash milestoneHash) { + unsolidMilestones.remove(milestoneHash); + solidificationQueue.remove(milestoneHash); + } - determineYoungestMilestoneInQueue(); - } + /** + * {@inheritDoc} + */ + @Override + public int getLatestMilestoneIndex(){ + return latestMilestoneIndex; } /** - *

- * This method contains the logic for the milestone solidification, that gets executed in a separate - * {@link Thread}. - *

- *

- * It executes the necessary steps periodically while waiting a short time to give the nodes the ability to - * answer to the issued transaction requests. - *

+ * {@inheritDoc} */ - private void milestoneSolidificationThread() { - processNewlyAddedMilestones(); - processSolidificationQueue(); - refillSolidificationQueue(); + @Override + public Hash getLatestMilestoneHash(){ + return latestMilestoneHash; } /** - *

- * This method processes the newly added milestones. - *

- *

- * We process them lazy to decrease the synchronization requirements and speed up the addition of milestones from - * outside {@link Thread}s. - *

- *

- * It iterates over the milestones and adds them to the pool. If they are older than the - * {@link #youngestMilestoneInQueue}, we add the to the solidification queue. - *

+ * {@inheritDoc} */ - private void processNewlyAddedMilestones() { - for (Iterator> iterator = newlyAddedMilestones.entrySet().iterator(); - !Thread.currentThread().isInterrupted() && iterator.hasNext();) { + @Override + public void setLatestMilestone(Hash latestMilestoneHash, int latestMilestoneIndex) { + this.latestMilestoneHash = latestMilestoneHash; + this.latestMilestoneIndex = latestMilestoneIndex; + } - Map.Entry currentEntry = iterator.next(); + /** + * {@inheritDoc} + */ + @Override + public boolean isInitialScanComplete(){ + return initialized; + } - unsolidMilestonesPool.put(currentEntry.getKey(), currentEntry.getValue()); + /** + * {@inheritDoc} + */ + @Override + public void logNewMilestone(int oldMilestoneIndex, int newMilestoneIndex, Hash newMilestoneHash){ + setLatestMilestone(newMilestoneHash, newMilestoneIndex); + tangle.publish("lmi %d %d", latestMilestoneIndex, newMilestoneIndex); + log.info("Latest milestone has changed from #" + oldMilestoneIndex + " to #" + newMilestoneIndex); + } - if (youngestMilestoneInQueue == null || currentEntry.getValue() < youngestMilestoneInQueue.getValue()) { - addToSolidificationQueue(currentEntry); + /** + * Remove milestone candidate from the {@link #seenMilestones} queue, then iterate through the + * {@link #unsolidMilestones} queue and remove any milestones with an index below the provided index. + * + * @param solidMilestoneIndex The milestone index to remove + */ + private void removeSeenMilestone(int solidMilestoneIndex) { + seenMilestones.remove(solidMilestoneIndex); + for (Iterator> iterator = unsolidMilestones.entrySet().iterator(); + !Thread.currentThread().isInterrupted() && iterator.hasNext();) { + Map.Entry nextMilestone = iterator.next(); + if (nextMilestone.getValue() <= solidMilestoneIndex || + nextMilestone.getValue() < snapshotProvider.getLatestSnapshot().getInitialIndex()) { + iterator.remove(); } - - iterator.remove(); } + scanMilestonesInQueue(); } /** - *

- * This method contains the logic for processing the {@link #milestonesToSolidify}. - *

- *

- * It iterates through the queue and checks if the corresponding milestones are still relevant for our node, or if - * they could be successfully solidified. If the milestones become solid or irrelevant, we remove them from the - * pool and the queue and reset the {@link #youngestMilestoneInQueue} marker (if necessary). - *

+ * Scan through milestones in the {@link #unsolidMilestones} queue and remove any that are present in the + * {@link #seenMilestones} queue. If it isn't in the {@link #seenMilestones} queue, then the youngest and oldest + * milestone objects are updated with that milestone object. Used to reset the oldest milestones for solidification. + * If upon scanning the {@link #unsolidMilestones} queue is empty and {@link #initialized} is false, set + * {@link #initialized} to true. */ - private void processSolidificationQueue() { - for (Iterator> iterator = milestonesToSolidify.entrySet().iterator(); - !Thread.currentThread().isInterrupted() && iterator.hasNext();) { - - Map.Entry currentEntry = iterator.next(); - - if (currentEntry.getValue() <= snapshotProvider.getInitialSnapshot().getIndex() || isSolid(currentEntry)) { - unsolidMilestonesPool.remove(currentEntry.getKey()); - iterator.remove(); + private void scanMilestonesInQueue() { + if(unsolidMilestones.size() > 0) { + Map newSolidificationQueue = new ConcurrentHashMap<>(); + unsolidMilestones.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .forEach(milestone -> { + if (newSolidificationQueue.size() < MAX_SIZE) { + newSolidificationQueue.put(milestone.getKey(), milestone.getValue()); + } + }); + solidificationQueue = newSolidificationQueue; + } + oldestMilestoneInQueue = null; + for (Map.Entry currentEntry : solidificationQueue.entrySet()) { + if (seenMilestones.containsKey(currentEntry.getValue())) { + unsolidMilestones.remove(currentEntry.getKey()); + solidificationQueue.remove(currentEntry.getKey()); + } else { + updateOldestMilestone(currentEntry.getKey(), currentEntry.getValue()); + } + } + if(!initialized && unsolidMilestones.size() == 0){ + initialized = true; + } + } - if (youngestMilestoneInQueue != null && - currentEntry.getKey().equals(youngestMilestoneInQueue.getKey())) { + private void updateOldestMilestone(Hash milestoneHash, int milestoneIndex){ + if (oldestMilestoneInQueue == null || oldestMilestoneInQueue.getValue() > milestoneIndex) { + oldestMilestoneInQueue = new AbstractMap.SimpleEntry<>(milestoneHash, milestoneIndex); + } + } - youngestMilestoneInQueue = null; - } + private void applySolidMilestoneToLedger(MilestoneViewModel milestone) throws Exception { + if (ledgerService.applyMilestoneToLedger(milestone)) { + if (isRepairRunning() && isRepairSuccessful(milestone)) { + stopRepair(); } + syncProgressInfo.addMilestoneApplicationTime(); + } else { + repairCorruptedMilestone(milestone); } } /** *

- * This method takes care of adding new milestones from the pool to the solidification queue, and filling it up - * again after it was processed / emptied before. + * Tries to actively repair the ledger by reverting the milestones preceding the given milestone. + *

+ *

+ * It gets called when a milestone could not be applied to the ledger state because of problems like "inconsistent + * balances". While this should theoretically never happen (because milestones are by definition "consistent"), it + * can still happen because IRI crashed or got stopped in the middle of applying a milestone or if a milestone + * was processed in the wrong order. + *

+ *

+ * Every time we call this method the internal {@link #repairBackoffCounter} is incremented which causes the next + * call of this method to repair an additional milestone. This means that whenever we face an error we first try to + * reset only the last milestone, then the two last milestones, then the three last milestones (and so on ...) until + * the problem was fixed. *

*

- * It first updates the {@link #youngestMilestoneInQueue} marker and then just adds new milestones as long as there - * is still space in the {@link #milestonesToSolidify} queue. + * To be able to tell when the problem is fixed and the {@link #repairBackoffCounter} can be reset, we store the + * milestone index that caused the problem the first time we call this method. *

+ * + * @param errorCausingMilestone the milestone that failed to be applied + * @throws MilestoneException if we failed to reset the corrupted milestone */ - private void refillSolidificationQueue() { - if(youngestMilestoneInQueue == null && !milestonesToSolidify.isEmpty()) { - determineYoungestMilestoneInQueue(); + private void repairCorruptedMilestone(MilestoneViewModel errorCausingMilestone) throws MilestoneException { + if (repairBackoffCounter++ == 0) { + errorCausingMilestoneIndex = errorCausingMilestone.index(); } - - Map.Entry nextSolidificationCandidate; - while (!Thread.currentThread().isInterrupted() && milestonesToSolidify.size() < SOLIDIFICATION_QUEUE_SIZE && - (nextSolidificationCandidate = getNextSolidificationCandidate()) != null) { - - addToSolidificationQueue(nextSolidificationCandidate); + for (int i = errorCausingMilestone.index(); i > errorCausingMilestone.index() - repairBackoffCounter; i--) { + milestoneService.resetCorruptedMilestone(i); } } /** *

- * This method determines the youngest milestone in the solidification queue. + * Checks if we are currently trying to repair a milestone. *

*

- * It iterates over all milestones in the Queue and keeps track of the youngest one found (the one with the highest - * milestone index). + * We simply use the {@link #repairBackoffCounter} as an indicator if a repair routine is running. *

+ * + * @return {@code true} if we are trying to repair a milestone and {@code false} otherwise */ - private void determineYoungestMilestoneInQueue() { - youngestMilestoneInQueue = null; - for (Map.Entry currentEntry : milestonesToSolidify.entrySet()) { - if (youngestMilestoneInQueue == null || currentEntry.getValue() > youngestMilestoneInQueue.getValue()) { - youngestMilestoneInQueue = currentEntry; - } - } + private boolean isRepairRunning() { + return repairBackoffCounter != 0; } /** *

- * This method returns the earliest seen Milestone from the unsolid milestones pool, that is not part of the - * {@link #milestonesToSolidify} queue yet. + * Checks if we successfully repaired the corrupted milestone. *

*

- * It simply iterates over all milestones in the pool and looks for the one with the lowest index, that is not - * getting actively solidified, yet. + * To determine if the repair routine was successful we check if the processed milestone has a higher index than the + * one that initially could not get applied to the ledger. *

* - * @return the Map.Entry holding the earliest milestone or null if the pool does not contain any new candidates. + * @param processedMilestone the currently processed milestone + * @return {@code true} if we advanced to a milestone following the corrupted one and {@code false} otherwise */ - private Map.Entry getNextSolidificationCandidate() { - Map.Entry nextSolidificationCandidate = null; - for (Map.Entry milestoneEntry : unsolidMilestonesPool.entrySet()) { - if (!milestonesToSolidify.containsKey(milestoneEntry.getKey()) && (nextSolidificationCandidate == null || - milestoneEntry.getValue() < nextSolidificationCandidate.getValue())) { - - nextSolidificationCandidate = milestoneEntry; - } - } - - return nextSolidificationCandidate; + private boolean isRepairSuccessful(MilestoneViewModel processedMilestone) { + return processedMilestone.index() > errorCausingMilestoneIndex; } /** *

- * This method performs the actual solidity check on the selected milestone. + * Resets the internal variables that are used to keep track of the repair process. *

*

- * It first dumps a log message to keep the node operator informed about the progress of solidification, and then - * issues the {@link TransactionSolidifier#checkSolidity(Hash, int)} call that starts the solidification - * process. + * It gets called whenever we advance to a milestone that has a higher milestone index than the milestone that + * initially caused the repair routine to kick in (see {@link #repairCorruptedMilestone(MilestoneViewModel)}. *

- *

- * We limit the amount of transactions that may be processed during the solidity check, since we want to solidify - * from the oldest milestone to the newest one and not "block" the solidification with a very recent milestone that - * needs to traverse huge chunks of the tangle. The main goal of this is to give the solidification just enough - * "resources" to discover the previous milestone while at the same time allowing fast solidity checks. - *

- * - * @param currentEntry milestone entry that shall be checked - * @return true if the given milestone is solid or false otherwise */ - private boolean isSolid(Map.Entry currentEntry) { - if (unsolidMilestonesPool.size() > 1) { - log.info("Solidifying milestone #" + currentEntry.getValue() + - " [" + milestonesToSolidify.size() + " / " + unsolidMilestonesPool.size() + "]"); + private void stopRepair() { + repairBackoffCounter = 0; + errorCausingMilestoneIndex = Integer.MAX_VALUE; + } + + + private void logChange(int prevSolidMilestoneIndex) { + Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); + int latestSolidMilestoneIndex = latestSnapshot.getIndex(); + + if (prevSolidMilestoneIndex == latestSolidMilestoneIndex) { + return; } - try { - return transactionSolidifier.addMilestoneToSolidificationQueue(currentEntry.getKey(), SOLIDIFICATION_TRANSACTIONS_LIMIT); - } catch (Exception e) { - log.error("Error while solidifying milestone #" + currentEntry.getValue(), e); + Hash latestMilestoneHash = latestSnapshot.getHash(); + log.delegate().info("Latest SOLID milestone index changed from #" + prevSolidMilestoneIndex + " to #" + latestSolidMilestoneIndex); - return false; + tangle.publish("lmsi %d %d", prevSolidMilestoneIndex, latestSolidMilestoneIndex); + tangle.publish("lmhs %s", latestMilestoneHash); + + if (!config.isPrintSyncProgressEnabled()) { + return; + } + + int latestMilestoneIndex = getLatestMilestoneIndex(); + + // only print more sophisticated progress if we are coming from a more unsynced state + if(latestMilestoneIndex - latestSolidMilestoneIndex < 1) { + syncProgressInfo.setSyncMilestoneStartIndex(latestSolidMilestoneIndex); + syncProgressInfo.resetMilestoneApplicationTimes(); + return; + } + + int estSecondsToBeSynced = syncProgressInfo.computeEstimatedTimeToSyncUpSeconds(latestMilestoneIndex, + latestSolidMilestoneIndex); + StringBuilder progressSB = new StringBuilder(); + + // add progress bar + progressSB.append(ASCIIProgressBar.getProgressBarString(syncProgressInfo.getSyncMilestoneStartIndex(), + latestMilestoneIndex, latestSolidMilestoneIndex)); + // add lsm to lm + progressSB.append(String.format(" [LSM %d / LM %d - remaining: %d]", latestSolidMilestoneIndex, + latestMilestoneIndex, latestMilestoneIndex - latestSolidMilestoneIndex)); + // add estimated time to get fully synced + if (estSecondsToBeSynced != -1) { + progressSB.append(String.format(" - est. seconds to get synced: %d", estSecondsToBeSynced)); } + log.info(progressSB.toString()); } + + + /** + * Holds variables containing information needed for sync progress calculation. + */ + private static class SyncProgressInfo { + /** + * The actual start milestone index from which the node started from when syncing up. + */ + private int syncMilestoneStartIndex; + + /** + * Used to calculate the average time needed to apply a milestone. + */ + private BlockingQueue lastMilestoneApplyTimes = new ArrayBlockingQueue<>(100); + + /** + * Gets the milestone sync start index. + * + * @return the milestone sync start index + */ + int getSyncMilestoneStartIndex() { + return syncMilestoneStartIndex; + } + + /** + * Sets the milestone sync start index. + * + * @param syncMilestoneStartIndex the value to set + */ + void setSyncMilestoneStartIndex(int syncMilestoneStartIndex) { + this.syncMilestoneStartIndex = syncMilestoneStartIndex; + } + + /** + * Adds the current time as a time where a milestone got applied to the ledger state. + */ + void addMilestoneApplicationTime(){ + if(lastMilestoneApplyTimes.remainingCapacity() == 0){ + lastMilestoneApplyTimes.remove(); + } + lastMilestoneApplyTimes.add(System.currentTimeMillis()); + } + + /** + * Clears the records of times when milestones got applied. + */ + void resetMilestoneApplicationTimes(){ + lastMilestoneApplyTimes.clear(); + } + + /** + * Computes the estimated time (seconds) needed to get synced up by looking at the last N milestone applications + * times and the current solid and last milestone indices. + * + * @param latestMilestoneIndex the current latest known milestone index + * @param latestSolidMilestoneIndex the current to the ledger applied milestone index + * @return the number of seconds needed to get synced up or -1 if not enough data is available + */ + int computeEstimatedTimeToSyncUpSeconds(int latestMilestoneIndex, int latestSolidMilestoneIndex) { + // compute average time needed to apply a milestone + Object[] times = lastMilestoneApplyTimes.toArray(); + long sumDelta = 0; + double avgMilestoneApplyMillisec; + if (times.length > 1) { + // compute delta sum + for (int i = times.length - 1; i > 1; i--) { + sumDelta += ((Long)times[i]) - ((Long)times[i - 1]); + } + avgMilestoneApplyMillisec = (double) sumDelta / (double) (times.length - 1); + } else { + return -1; + } + + return (int) ((avgMilestoneApplyMillisec / 1000) * (latestMilestoneIndex - latestSolidMilestoneIndex)); + } + } + } diff --git a/src/main/java/com/iota/iri/service/snapshot/LocalSnapshotManager.java b/src/main/java/com/iota/iri/service/snapshot/LocalSnapshotManager.java index ba760f846b..3f7a2cd9cc 100644 --- a/src/main/java/com/iota/iri/service/snapshot/LocalSnapshotManager.java +++ b/src/main/java/com/iota/iri/service/snapshot/LocalSnapshotManager.java @@ -1,6 +1,6 @@ package com.iota.iri.service.snapshot; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; /** * Represents the manager for local {@link Snapshot}s that takes care of periodically creating a new {@link Snapshot} @@ -17,13 +17,13 @@ public interface LocalSnapshotManager { * Note: If the node is not fully synced we use * {@link com.iota.iri.conf.SnapshotConfig#getLocalSnapshotsIntervalUnsynced()} instead. * - * @param latestMilestoneTracker tracker for the milestones to determine when a new local snapshot is due + * @param milestoneSolidifier tracker for the milestones to determine when a new local snapshot is due */ - void start(LatestMilestoneTracker latestMilestoneTracker); + void start(MilestoneSolidifier milestoneSolidifier); /** * Stops the {@link Thread} that takes care of creating the local {@link Snapshot}s and that was spawned by the - * {@link #start(LatestMilestoneTracker)} method. + * {@link #start(MilestoneSolidifier)} method. */ void shutdown(); } diff --git a/src/main/java/com/iota/iri/service/snapshot/SnapshotService.java b/src/main/java/com/iota/iri/service/snapshot/SnapshotService.java index cdd7592c2a..41e5223674 100644 --- a/src/main/java/com/iota/iri/service/snapshot/SnapshotService.java +++ b/src/main/java/com/iota/iri/service/snapshot/SnapshotService.java @@ -2,7 +2,7 @@ import com.iota.iri.controllers.MilestoneViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.transactionpruning.TransactionPruner; import java.util.Map; @@ -71,11 +71,11 @@ public interface SnapshotService { * by the {@code snapshotProvider} to reflect the newly created {@link Snapshot}. *

* - * @param latestMilestoneTracker milestone tracker that allows us to retrieve information about the known milestones + * @param milestoneSolidifier milestone tracker that allows us to retrieve information about the known milestones * @param transactionPruner manager for the pruning jobs that takes care of cleaning up the old data that * @throws SnapshotException if anything goes wrong while creating the local snapshot */ - void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, TransactionPruner transactionPruner) throws + void takeLocalSnapshot(MilestoneSolidifier milestoneSolidifier, TransactionPruner transactionPruner) throws SnapshotException; /** @@ -88,12 +88,12 @@ void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, Transactio * local snapshot files. *

* - * @param latestMilestoneTracker milestone tracker that allows us to retrieve information about the known milestones + * @param milestoneSolidifier milestone tracker that allows us to retrieve information about the known milestones * @param targetMilestone milestone that is used as a reference point for the snapshot * @return a local snapshot of the full ledger state at the given milestone * @throws SnapshotException if anything goes wrong while generating the local snapshot */ - Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, MilestoneViewModel targetMilestone) throws + Snapshot generateSnapshot(MilestoneSolidifier milestoneSolidifier, MilestoneViewModel targetMilestone) throws SnapshotException; /** @@ -122,11 +122,11 @@ Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, Milesto * very first time. *

* - * @param latestMilestoneTracker milestone tracker that allows us to retrieve information about the known milestones + * @param milestoneSolidifier milestone tracker that allows us to retrieve information about the known milestones * @param targetMilestone milestone that is used as a reference point for the snapshot * @return a map of solid entry points associating their hash to the milestone index that confirmed them * @throws SnapshotException if anything goes wrong while generating the solid entry points */ - Map generateSeenMilestones(LatestMilestoneTracker latestMilestoneTracker, - MilestoneViewModel targetMilestone) throws SnapshotException; + Map generateSeenMilestones(MilestoneSolidifier milestoneSolidifier, + MilestoneViewModel targetMilestone) throws SnapshotException; } diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImpl.java index 775d1ed40e..1f0e62d818 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImpl.java @@ -2,7 +2,7 @@ import com.google.common.annotations.VisibleForTesting; import com.iota.iri.conf.SnapshotConfig; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.LocalSnapshotManager; import com.iota.iri.service.snapshot.SnapshotService; import com.iota.iri.service.snapshot.SnapshotException; @@ -21,7 +21,7 @@ *

*

* It incorporates a background worker that periodically checks if a new snapshot is due (see {@link - * #start(LatestMilestoneTracker)} and {@link #shutdown()}). + * #start(MilestoneSolidifier)} and {@link #shutdown()}). *

*/ public class LocalSnapshotManagerImpl implements LocalSnapshotManager { @@ -72,7 +72,7 @@ public class LocalSnapshotManagerImpl implements LocalSnapshotManager { * Holds a reference to the {@link ThreadIdentifier} for the monitor thread. * * Using a {@link ThreadIdentifier} for spawning the thread allows the {@link ThreadUtils} to spawn exactly one - * thread for this instance even when we call the {@link #start(LatestMilestoneTracker)} method multiple times. + * thread for this instance even when we call the {@link #start(MilestoneSolidifier)} method multiple times. */ private ThreadIdentifier monitorThreadIdentifier = new ThreadIdentifier("Local Snapshots Monitor"); @@ -95,8 +95,8 @@ public LocalSnapshotManagerImpl(SnapshotProvider snapshotProvider, SnapshotServi * {@inheritDoc} */ @Override - public void start(LatestMilestoneTracker latestMilestoneTracker) { - ThreadUtils.spawnThread(() -> monitorThread(latestMilestoneTracker), monitorThreadIdentifier); + public void start(MilestoneSolidifier milestoneSolidifier) { + ThreadUtils.spawnThread(() -> monitorThread(milestoneSolidifier), monitorThreadIdentifier); } /** @@ -113,21 +113,21 @@ public void shutdown() { * It periodically checks if a new {@link com.iota.iri.service.snapshot.Snapshot} has to be taken until the * {@link Thread} is terminated. If it detects that a {@link com.iota.iri.service.snapshot.Snapshot} is due it * triggers the creation of the {@link com.iota.iri.service.snapshot.Snapshot} by calling - * {@link SnapshotService#takeLocalSnapshot(LatestMilestoneTracker, TransactionPruner)}. + * {@link SnapshotService#takeLocalSnapshot(MilestoneSolidifier, TransactionPruner)}. * - * @param latestMilestoneTracker tracker for the milestones to determine when a new local snapshot is due + * @param milestoneSolidifier tracker for the milestones to determine when a new local snapshot is due */ @VisibleForTesting - void monitorThread(LatestMilestoneTracker latestMilestoneTracker) { + void monitorThread(MilestoneSolidifier milestoneSolidifier) { while (!Thread.currentThread().isInterrupted()) { - int localSnapshotInterval = getSnapshotInterval(isInSync(latestMilestoneTracker)); + int localSnapshotInterval = getSnapshotInterval(isInSync(milestoneSolidifier)); int latestSnapshotIndex = snapshotProvider.getLatestSnapshot().getIndex(); int initialSnapshotIndex = snapshotProvider.getInitialSnapshot().getIndex(); if (latestSnapshotIndex - initialSnapshotIndex > config.getLocalSnapshotsDepth() + localSnapshotInterval) { try { - snapshotService.takeLocalSnapshot(latestMilestoneTracker, transactionPruner); + snapshotService.takeLocalSnapshot(milestoneSolidifier, transactionPruner); } catch (SnapshotException e) { log.error("error while taking local snapshot", e); } @@ -159,16 +159,16 @@ int getSnapshotInterval(boolean inSync) { * This will always return false if we are not done scanning milestone * candidates during initialization. * - * @param latestMilestoneTracker tracker we use to determine milestones + * @param milestoneSolidifier tracker we use to determine milestones * @return true if we are in sync, otherwise false */ @VisibleForTesting - boolean isInSync(LatestMilestoneTracker latestMilestoneTracker) { - if (!latestMilestoneTracker.isInitialScanComplete()) { + boolean isInSync(MilestoneSolidifier milestoneSolidifier) { + if (!milestoneSolidifier.isInitialScanComplete()) { return false; } - int latestIndex = latestMilestoneTracker.getLatestMilestoneIndex(); + int latestIndex = milestoneSolidifier.getLatestMilestoneIndex(); int latestSnapshot = snapshotProvider.getLatestSnapshot().getIndex(); // If we are out of sync, only a full sync will get us in diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java index 332c23546e..cdcf27d2c2 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java @@ -6,7 +6,7 @@ import com.iota.iri.controllers.StateDiffViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.*; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPruningException; @@ -196,12 +196,12 @@ public void rollBackMilestones(Snapshot snapshot, int targetMilestoneIndex) thro * {@inheritDoc} */ @Override - public void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, TransactionPruner transactionPruner) + public void takeLocalSnapshot(MilestoneSolidifier milestoneSolidifier, TransactionPruner transactionPruner) throws SnapshotException { MilestoneViewModel targetMilestone = determineMilestoneForLocalSnapshot(tangle, snapshotProvider, config); - Snapshot newSnapshot = generateSnapshot(latestMilestoneTracker, targetMilestone); + Snapshot newSnapshot = generateSnapshot(milestoneSolidifier, targetMilestone); if (transactionPruner != null) { cleanupExpiredSolidEntryPoints(tangle, snapshotProvider.getInitialSnapshot().getSolidEntryPoints(), @@ -217,7 +217,7 @@ public void takeLocalSnapshot(LatestMilestoneTracker latestMilestoneTracker, Tra * {@inheritDoc} */ @Override - public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, MilestoneViewModel targetMilestone) + public Snapshot generateSnapshot(MilestoneSolidifier milestoneSolidifier, MilestoneViewModel targetMilestone) throws SnapshotException { if (targetMilestone == null) { @@ -253,7 +253,7 @@ public Snapshot generateSnapshot(LatestMilestoneTracker latestMilestoneTracker, } snapshot.setSolidEntryPoints(generateSolidEntryPoints(targetMilestone)); - snapshot.setSeenMilestones(generateSeenMilestones(latestMilestoneTracker, targetMilestone)); + snapshot.setSeenMilestones(generateSeenMilestones(milestoneSolidifier, targetMilestone)); return snapshot; } @@ -276,8 +276,8 @@ public Map generateSolidEntryPoints(MilestoneViewModel targetMile * {@inheritDoc} */ @Override - public Map generateSeenMilestones(LatestMilestoneTracker latestMilestoneTracker, - MilestoneViewModel targetMilestone) throws SnapshotException { + public Map generateSeenMilestones(MilestoneSolidifier milestoneSolidifier, + MilestoneViewModel targetMilestone) throws SnapshotException { ProgressLogger progressLogger = new IntervalProgressLogger( "Taking local snapshot [processing seen milestones]", log) @@ -287,7 +287,7 @@ public Map generateSeenMilestones(LatestMilestoneTracker latestMi try { MilestoneViewModel seenMilestone = targetMilestone; while ((seenMilestone = MilestoneViewModel.findClosestNextMilestone(tangle, seenMilestone.index(), - latestMilestoneTracker.getLatestMilestoneIndex())) != null) { + milestoneSolidifier.getLatestMilestoneIndex())) != null) { seenMilestones.put(seenMilestone.getHash(), seenMilestone.index()); diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java index 8e80247c2a..bfa9d0a332 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java @@ -2,7 +2,7 @@ import com.iota.iri.controllers.MilestoneViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.tipselection.EntryPointSelector; import com.iota.iri.storage.Tangle; @@ -16,19 +16,19 @@ public class EntryPointSelectorImpl implements EntryPointSelector { private final Tangle tangle; private final SnapshotProvider snapshotProvider; - private final LatestMilestoneTracker latestMilestoneTracker; + private final MilestoneSolidifier milestoneSolidifier; /** * Constructor for Entry Point Selector * @param tangle Tangle object which acts as a database interface. * @param snapshotProvider accesses snapshots of the ledger state - * @param latestMilestoneTracker used to get latest milestone. + * @param milestoneSolidifier used to get latest milestone. */ public EntryPointSelectorImpl(Tangle tangle, SnapshotProvider snapshotProvider, - LatestMilestoneTracker latestMilestoneTracker) { + MilestoneSolidifier milestoneSolidifier) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; - this.latestMilestoneTracker = latestMilestoneTracker; + this.milestoneSolidifier = milestoneSolidifier; } @Override @@ -36,7 +36,7 @@ public Hash getEntryPoint(int depth) throws Exception { int milestoneIndex = Math.max(snapshotProvider.getLatestSnapshot().getIndex() - depth - 1, snapshotProvider.getInitialSnapshot().getIndex()); MilestoneViewModel milestoneViewModel = MilestoneViewModel.findClosestNextMilestone(tangle, milestoneIndex, - latestMilestoneTracker.getLatestMilestoneIndex()); + milestoneSolidifier.getLatestMilestoneIndex()); if (milestoneViewModel != null && milestoneViewModel.getHash() != null) { return milestoneViewModel.getHash(); } diff --git a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java index 9899ec4f9d..3e2ed5bec5 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionSolidifier.java @@ -1,7 +1,9 @@ package com.iota.iri.service.validation; +import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +import com.iota.iri.service.validation.impl.TransactionSolidifierImpl; import com.iota.iri.network.TransactionRequester; import java.util.Set; @@ -81,4 +83,44 @@ public interface TransactionSolidifier { * @throws Exception if anything goes wrong while trying to solidify the transaction */ boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception; + + /** + * Updates a transaction after it was stored in the tangle. Tells the node to not request the transaction anymore, + * to update the live tips accordingly, and attempts to quickly solidify the transaction. + * + *

+ * Performs the following operations: + * + *

    + *
  1. Removes {@code transactionViewModel}'s hash from the the request queue since we already found it.
    2. + *
  2. If {@code transactionViewModel} has no children (approvers), we add it to the node's active tip list.
    3. + *
  3. Removes {@code transactionViewModel}'s parents (branch & trunk) from the node's tip list + * (if they're present there).
    4. + *
  4. Attempts to quickly solidify {@code transactionViewModel} by checking whether its direct parents + * are solid. If solid we add it to the queue transaction solidification thread to help it propagate the + * solidification to the approving child transactions.
    5. + *
  5. Requests missing direct parent (trunk & branch) transactions that are needed to solidify + * {@code transactionViewModel}.
    6. + *
+ * @param transactionViewModel received transaction that is being updated + * @throws Exception if an error occurred while trying to solidify + * @see TipsViewModel + */ + void updateStatus(TransactionViewModel transactionViewModel) throws Exception; + + /** + * Tries to solidify the transactions quickly by performing {@link TransactionSolidifierImpl#checkApproovee} on + * both parents (trunk and branch). If the parents are solid, mark the transactions as solid. + * @param transactionViewModel transaction to solidify + * @return true if we made the transaction solid, else false. + * @throws Exception + */ + boolean quickSetSolid(TransactionViewModel transactionViewModel) throws Exception; + + /** + * Add to the propagation queue where it will be processed to help solidify approving transactions faster + * @param hash The transaction hash to be removed + * @throws Exception + */ + void addToPropagationQueue(Hash hash) throws Exception; } diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index 0a59441c50..3350ecbaa7 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -160,88 +160,6 @@ public TransactionViewModel validateBytes(final byte[] bytes, int minWeightMagni return transactionViewModel; } - /** - * Updates a transaction after it was stored in the tangle. Tells the node to not request the transaction anymore, - * to update the live tips accordingly, and attempts to quickly solidify the transaction. - * - *

- * Performs the following operations: - * - *

    - *
  1. Removes {@code transactionViewModel}'s hash from the the request queue since we already found it.
    2. - *
  2. If {@code transactionViewModel} has no children (approvers), we add it to the node's active tip list.
    3. - *
  3. Removes {@code transactionViewModel}'s parents (branch & trunk) from the node's tip list - * (if they're present there).
    4. - *
  4. Attempts to quickly solidify {@code transactionViewModel} by checking whether its direct parents - * are solid. If solid we add it to the queue transaction solidification thread to help it propagate the - * solidification to the approving child transactions.
    5. - *
  5. Requests missing direct parent (trunk & branch) transactions that are needed to solidify - * {@code transactionViewModel}.
    6. - *
- * @param transactionViewModel received transaction that is being updated - * @throws Exception if an error occurred while trying to solidify - * @see TipsViewModel - */ - //Not part of the validation process. This should be moved to a component in charge of - //what transaction we gossip. - public void updateStatus(TransactionViewModel transactionViewModel) throws Exception { - transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); - if(transactionViewModel.getApprovers(tangle).size() == 0) { - tipsViewModel.addTipHash(transactionViewModel.getHash()); - } - tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); - tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); - - if(quickSetSolid(transactionViewModel)) { - transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); - tipsViewModel.setSolid(transactionViewModel.getHash()); - } - } - - /** - * Tries to solidify the transactions quickly by performing {@link #checkApproovee} on both parents (trunk and - * branch). If the parents are solid, mark the transactions as solid. - * @param transactionViewModel transaction to solidify - * @return true if we made the transaction solid, else false. - * @throws Exception - */ - public boolean quickSetSolid(final TransactionViewModel transactionViewModel) throws Exception { - if(!transactionViewModel.isSolid()) { - boolean solid = true; - if (!checkApproovee(transactionViewModel.getTrunkTransaction(tangle))) { - solid = false; - } - if (!checkApproovee(transactionViewModel.getBranchTransaction(tangle))) { - solid = false; - } - if(solid) { - transactionViewModel.updateSolid(true); - transactionViewModel.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); - return true; - } - } - return false; - } - - /** - * If the the {@code approvee} is missing, request it from a neighbor. - * @param approovee transaction we check. - * @return true if {@code approvee} is solid. - * @throws Exception if we encounter an error while requesting a transaction - */ - private boolean checkApproovee(TransactionViewModel approovee) throws Exception { - if(snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(approovee.getHash())) { - return true; - } - if(approovee.getType() == PREFILLED_SLOT) { - // don't solidify from the bottom until cuckoo filters can identify where we deleted -> otherwise we will - // continue requesting old transactions forever - //transactionRequester.requestTransaction(approovee.getHash(), false); - return false; - } - return approovee.isSolid(); - } - /** * Thrown if transaction fails {@link #hasInvalidTimestamp} check. */ diff --git a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java index 924d025ebf..f040f94a57 100644 --- a/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/validation/impl/TransactionSolidifierImpl.java @@ -1,6 +1,7 @@ package com.iota.iri.service.validation.impl; import com.google.common.annotations.VisibleForTesting; +import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; @@ -32,7 +33,7 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { /** * Max size for all queues. */ - private static final int MAX_SIZE= 10000; + private static final int MAX_SIZE= 100; private static final IntervalLogger log = new IntervalLogger(TransactionSolidifier.class); @@ -44,19 +45,24 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { /** * Interval that the {@link #processTransactionsToSolidify()} will scan at. */ - private static final int RESCAN_INTERVAL = 500; + private static final int RESCAN_INTERVAL = 50; /** * A queue for processing transactions with the {@link #checkSolidity(Hash)} call. Once a transaction has been * marked solid it will be placed into the {@link #transactionsToBroadcast} queue. */ private BlockingQueue transactionsToSolidify = new ArrayBlockingQueue<>(MAX_SIZE); + private BlockingQueue solidTransactions = new ArrayBlockingQueue<>(MAX_SIZE); + /** * A set of transactions that will be called by the {@link TransactionProcessingPipeline} to be broadcast to * neighboring nodes. */ private BlockingQueue transactionsToBroadcast = new ArrayBlockingQueue<>(MAX_SIZE); + private TipsViewModel tipsViewModel; + + private Hash cooAddress; /** * Constructor for the solidifier. @@ -64,10 +70,13 @@ public class TransactionSolidifierImpl implements TransactionSolidifier { * @param snapshotProvider For fetching entry points for solidity checks * @param transactionRequester A requester for missing transactions */ - public TransactionSolidifierImpl(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester){ + public TransactionSolidifierImpl(Tangle tangle, SnapshotProvider snapshotProvider, TransactionRequester transactionRequester, + TipsViewModel tipsViewModel, Hash cooAddress){ this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.transactionRequester = transactionRequester; + this.tipsViewModel = tipsViewModel; + this.cooAddress = cooAddress; } /** @@ -75,8 +84,7 @@ public TransactionSolidifierImpl(Tangle tangle, SnapshotProvider snapshotProvide */ @Override public void start(){ - executorService.silentScheduleWithFixedDelay(this::processTransactionsToSolidify, 0, RESCAN_INTERVAL, - TimeUnit.MILLISECONDS); + executorService.silentExecute(this::processTransactionsToSolidify); } @@ -94,12 +102,13 @@ public void shutdown() { @Override public void addToSolidificationQueue(Hash hash){ try{ - if(transactionsToSolidify.size() >= MAX_SIZE - 1){ - transactionsToSolidify.remove(); - } - if(!transactionsToSolidify.contains(hash)) { - transactionsToSolidify.put(hash); + + if(transactionsToSolidify.size() >= MAX_SIZE - 1){ + transactionsToSolidify.remove(); + } + + transactionsToSolidify.put(hash); } } catch(Exception e){ log.error("Error placing transaction into solidification queue",e); @@ -111,6 +120,7 @@ public boolean addMilestoneToSolidificationQueue(Hash hash, int maxToProcess){ try{ TransactionViewModel tx = fromHash(tangle, hash); if(tx.isSolid()){ + addToPropagationQueue(hash); return true; } addToSolidificationQueue(hash); @@ -119,7 +129,6 @@ public boolean addMilestoneToSolidificationQueue(Hash hash, int maxToProcess){ log.error("Error adding milestone to solidification queue", e); return false; } - } /** @@ -155,6 +164,7 @@ private void processTransactionsToSolidify(){ log.info(e.getMessage()); } } + propagateSolidTransactions(); } } @@ -163,7 +173,7 @@ private void processTransactionsToSolidify(){ */ @Override public boolean checkSolidity(Hash hash) throws Exception { - return checkSolidity(hash, Integer.MAX_VALUE); + return checkSolidity(hash, 50000); } /** @@ -201,6 +211,7 @@ public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exc } } } + if (solid) { updateTransactions(analyzedHashes); } @@ -234,6 +245,7 @@ private void updateTransactions(Set hashes) { tvm.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); } addToBroadcastQueue(tvm); + addToPropagationQueue(tvm.getHash()); } catch (Exception e) { log.info(e.getMessage()); } @@ -243,8 +255,14 @@ private void updateTransactions(Set hashes) { /** * Returns true if transaction is not solid and there are no solid entry points from the initial snapshot. */ - private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Hash hashPointer){ - return (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)); + private boolean isUnsolidWithoutEntryPoint(TransactionViewModel transaction, Hash hashPointer) throws Exception{ + if(!transaction.isSolid()){ + if(!snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)){ + return true; + } + } + addToPropagationQueue(hashPointer); + return false; } @@ -264,4 +282,108 @@ private void addToBroadcastQueue(TransactionViewModel tvm) { Set getSolidificationQueue(){ return new LinkedHashSet<>(transactionsToSolidify); } + + + @Override + public void updateStatus(TransactionViewModel transactionViewModel) throws Exception { + transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); + if(transactionViewModel.getApprovers(tangle).size() == 0) { + tipsViewModel.addTipHash(transactionViewModel.getHash()); + } + tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); + tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); + + if(quickSetSolid(transactionViewModel)) { + transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); + tipsViewModel.setSolid(transactionViewModel.getHash()); + addToPropagationQueue(transactionViewModel.getHash()); + } + } + + public void addToPropagationQueue(Hash hash) throws Exception{ + if(!solidTransactions.contains(hash)) { + if (solidTransactions.size() >= MAX_SIZE) { + solidTransactions.poll(); + } + solidTransactions.put(hash); + } + } + + @Override + public boolean quickSetSolid(final TransactionViewModel transactionViewModel) throws Exception { + if(!transactionViewModel.isSolid()) { + boolean solid = true; + if (!checkApproovee(transactionViewModel.getTrunkTransaction(tangle))) { + solid = false; + } + if (!checkApproovee(transactionViewModel.getBranchTransaction(tangle))) { + solid = false; + } + if(solid) { + transactionViewModel.updateSolid(true); + transactionViewModel.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); + addToPropagationQueue(transactionViewModel.getHash()); + addToBroadcastQueue(transactionViewModel); + return true; + } + } + return false; + } + + /** + * If the the {@code approvee} is missing, request it from a neighbor. + * @param approovee transaction we check. + * @return true if {@code approvee} is solid. + * @throws Exception if we encounter an error while requesting a transaction + */ + private boolean checkApproovee(TransactionViewModel approovee) throws Exception { + if(snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(approovee.getHash())) { + return true; + } + if(approovee.getType() == PREFILLED_SLOT) { + // don't solidify from the bottom until cuckoo filters can identify where we deleted -> otherwise we will + // continue requesting old transactions forever + //transactionRequester.requestTransaction(approovee.getHash(), false); + return false; + } + return approovee.isSolid(); + } + + @VisibleForTesting + void propagateSolidTransactions() { + Iterator cascadeIterator = solidTransactions.iterator(); + int cascadeCount = 0; + while(cascadeCount < MAX_SIZE && cascadeIterator.hasNext() && !Thread.currentThread().isInterrupted()) { + try { + cascadeCount += 1; + Hash hash = cascadeIterator.next(); + TransactionViewModel transaction = fromHash(tangle, hash); + Set approvers = transaction.getApprovers(tangle).getHashes(); + for(Hash h: approvers) { + TransactionViewModel tx = fromHash(tangle, h); + if (quietQuickSetSolid(tx)) { + tx.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); + tipsViewModel.setSolid(h); + } + } + cascadeIterator.remove(); + } catch (Exception e) { + log.error("Error while propagating solidity upwards", e); + } + } + } + + /** + * Perform a {@link #quickSetSolid} while capturing and logging errors + * @param transactionViewModel transaction we try to solidify. + * @return true if we managed to solidify, else false. + */ + private boolean quietQuickSetSolid(TransactionViewModel transactionViewModel) { + try { + return quickSetSolid(transactionViewModel); + } catch (Exception e) { + log.error(e.getMessage(), e); + return false; + } + } } diff --git a/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java b/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java index 911bd9a63b..e0f1f99cf1 100644 --- a/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java +++ b/src/test/java/com/iota/iri/MainInjectionConfigurationTest.java @@ -12,8 +12,6 @@ import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.API; import com.iota.iri.service.ledger.LedgerService; -import com.iota.iri.service.milestone.LatestMilestoneTracker; -import com.iota.iri.service.milestone.LatestSolidMilestoneTracker; import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.milestone.SeenMilestonesRetriever; @@ -64,16 +62,6 @@ public void provideLedgerService() { assertNotNull("instance creation did not work", testInjector().getInstance(LedgerService.class)); } - @Test - public void provideLatestMilestoneTracker() { - assertNotNull("instance creation did not work", testInjector().getInstance(LatestMilestoneTracker.class)); - } - - @Test - public void provideLatestSolidMilestoneTracker() { - assertNotNull("instance creation did not work", testInjector().getInstance(LatestSolidMilestoneTracker.class)); - } - @Test public void provideSeenMilestonesRetriever() { assertNotNull("instance creation did not work", testInjector().getInstance(SeenMilestonesRetriever.class)); diff --git a/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java b/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java index 5affbf062e..a3333d0c04 100644 --- a/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java +++ b/src/test/java/com/iota/iri/network/NetworkInjectionConfigurationTest.java @@ -3,12 +3,13 @@ import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.BaseIotaConfig; import com.iota.iri.conf.IotaConfig; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import org.junit.Test; @@ -53,10 +54,11 @@ private class MockedDependencies extends AbstractModule { @Override protected void configure() { - bind(LatestMilestoneTracker.class).toInstance(mock(LatestMilestoneTracker.class)); + bind(MilestoneSolidifier.class).toInstance(mock(MilestoneSolidifier.class)); bind(SnapshotProvider.class).toInstance(mock(SnapshotProvider.class)); bind(TransactionValidator.class).toInstance(mock(TransactionValidator.class)); bind(TransactionSolidifier.class).toInstance(mock(TransactionSolidifier.class)); + bind(MilestoneService.class).toInstance(mock(MilestoneService.class)); } } diff --git a/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java b/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java index db75b68018..b6a40412c2 100644 --- a/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/ReceivedStageTest.java @@ -1,11 +1,13 @@ package com.iota.iri.network.pipeline; -import com.iota.iri.service.validation.TransactionValidator; +import com.iota.iri.model.Hash; +import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.Neighbor; import com.iota.iri.network.neighbor.impl.NeighborMetricsImpl; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.storage.Tangle; import org.junit.Rule; @@ -26,7 +28,7 @@ public class ReceivedStageTest { private Tangle tangle; @Mock - private TransactionValidator transactionValidator; + private TransactionSolidifier txSolidifier; @Mock private SnapshotProvider snapshotProvider; @@ -43,13 +45,19 @@ public class ReceivedStageTest { @Mock private NeighborMetricsImpl neighborMetrics; + @Mock + private MilestoneService milestoneService; + + @Mock + private Hash cooAddress; + @Test public void newlyStoredTransactionUpdatesAlsoArrivalTimeAndSender() throws Exception { Mockito.when(tvm.store(tangle, snapshotProvider.getInitialSnapshot())).thenReturn(true); Mockito.when(neighbor.getMetrics()).thenReturn(neighborMetrics); Mockito.when(transactionRequester.removeRecentlyRequestedTransaction(Mockito.any())).thenReturn(true); - ReceivedStage stage = new ReceivedStage(tangle, transactionValidator, snapshotProvider, transactionRequester); + ReceivedStage stage = new ReceivedStage(tangle, txSolidifier, snapshotProvider, transactionRequester, milestoneService, cooAddress); ReceivedPayload receivedPayload = new ReceivedPayload(neighbor, tvm); ProcessingContext ctx = new ProcessingContext(null, receivedPayload); stage.process(ctx); @@ -70,7 +78,7 @@ public void alreadyStoredTransactionDoesNoUpdates() throws Exception { Mockito.when(tvm.store(tangle, snapshotProvider.getInitialSnapshot())).thenReturn(false); Mockito.when(neighbor.getMetrics()).thenReturn(neighborMetrics); - ReceivedStage stage = new ReceivedStage(tangle, transactionValidator, snapshotProvider, transactionRequester); + ReceivedStage stage = new ReceivedStage(tangle, txSolidifier, snapshotProvider, transactionRequester, milestoneService, cooAddress); ReceivedPayload receivedPayload = new ReceivedPayload(neighbor, tvm); ProcessingContext ctx = new ProcessingContext(null, receivedPayload); stage.process(ctx); diff --git a/src/test/java/com/iota/iri/network/pipeline/ReplyStageTest.java b/src/test/java/com/iota/iri/network/pipeline/ReplyStageTest.java index fc1ba0f2bc..96eff3009b 100644 --- a/src/test/java/com/iota/iri/network/pipeline/ReplyStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/ReplyStageTest.java @@ -10,7 +10,7 @@ import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.impl.NeighborImpl; import com.iota.iri.network.neighbor.impl.NeighborMetricsImpl; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; @@ -44,7 +44,7 @@ public class ReplyStageTest { private TipsViewModel tipsViewModel; @Mock - private LatestMilestoneTracker latestMilestoneTracker; + private MilestoneSolidifier milestoneSolidifier; @Mock private SnapshotProvider snapshotProvider; @@ -74,13 +74,13 @@ public void usingTheNullHashARandomTipIsGettingReplied() { Mockito.when(neighbor.getMetrics()).thenReturn(neighborMetrics); Mockito.when(snapshotProvider.getLatestSnapshot()).thenReturn(snapshot); Mockito.when(snapshot.getIndex()).thenReturn(8); - Mockito.when(latestMilestoneTracker.getLatestMilestoneIndex()).thenReturn(10); + Mockito.when(milestoneSolidifier.getLatestMilestoneIndex()).thenReturn(10); Mockito.when(transactionRequester.numberOfTransactionsToRequest()).thenReturn(1); Mockito.when(tipsViewModel.getRandomSolidTipHash()).thenReturn(SampleTransaction.CURL_HASH_OF_SAMPLE_TX); TangleMockUtils.mockTransaction(tangle, SampleTransaction.CURL_HASH_OF_SAMPLE_TX, SampleTransaction.SAMPLE_TRANSACTION); - ReplyStage stage = new ReplyStage(neighborRouter, nodeConfig, tangle, tipsViewModel, latestMilestoneTracker, + ReplyStage stage = new ReplyStage(neighborRouter, nodeConfig, tangle, tipsViewModel, milestoneSolidifier, snapshotProvider, recentlySeenBytesCache, random); ReplyPayload replyPayload = new ReplyPayload(neighbor, Hash.NULL_HASH); ProcessingContext ctx = new ProcessingContext(replyPayload); @@ -100,7 +100,7 @@ public void usingASuppliedRequestedHashTheTransactionIsReplied() { TangleMockUtils.mockTransaction(tangle, SampleTransaction.CURL_HASH_OF_SAMPLE_TX, SampleTransaction.SAMPLE_TRANSACTION); - ReplyStage stage = new ReplyStage(neighborRouter, nodeConfig, tangle, tipsViewModel, latestMilestoneTracker, + ReplyStage stage = new ReplyStage(neighborRouter, nodeConfig, tangle, tipsViewModel, milestoneSolidifier, snapshotProvider, recentlySeenBytesCache, random); ReplyPayload replyPayload = new ReplyPayload(neighbor, SampleTransaction.CURL_HASH_OF_SAMPLE_TX); ProcessingContext ctx = new ProcessingContext(replyPayload); diff --git a/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java b/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java index 43830c6d8f..e79e27ccce 100644 --- a/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java @@ -38,15 +38,12 @@ public class SolidifyStageTest { @Mock private TransactionSolidifier transactionSolidifier; - @Mock - private TransactionValidator transactionValidator; - @Test public void solidTransactionIsBroadcast() throws Exception{ Mockito.when(tvm.isSolid()).thenReturn(true); Mockito.when(tvm.getHash()).thenReturn(originalHash); - SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, transactionValidator, tipsViewModel, tangle); + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, tipsViewModel, tangle); SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); ProcessingContext ctx = new ProcessingContext(solidifyPayload); @@ -62,10 +59,10 @@ public void solidTransactionIsBroadcast() throws Exception{ @Test public void quickSetSolidTransactionIsBroadcast() throws Exception{ - Mockito.when(transactionValidator.quickSetSolid(tvm)).thenReturn(true); + Mockito.when(transactionSolidifier.quickSetSolid(tvm)).thenReturn(true); Mockito.when(tvm.getHash()).thenReturn(originalHash); - SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, transactionValidator, tipsViewModel, tangle); + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, tipsViewModel, tangle); SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); ProcessingContext ctx = new ProcessingContext(solidifyPayload); @@ -82,10 +79,10 @@ public void quickSetSolidTransactionIsBroadcast() throws Exception{ @Test public void unsolidTransactionBroadcastsRandomSolidTip() throws Exception{ Mockito.when(tvm.isSolid()).thenReturn(false); - Mockito.when(transactionValidator.quickSetSolid(tvm)).thenReturn(false); + Mockito.when(transactionSolidifier.quickSetSolid(tvm)).thenReturn(false); TransactionViewModel tip = new TransactionViewModel(new Transaction(), tipHash); - SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, transactionValidator, tipsViewModel, tangle); + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, tipsViewModel, tangle); SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); ProcessingContext ctx = new ProcessingContext(solidifyPayload); @@ -103,10 +100,10 @@ public void unsolidTransactionBroadcastsRandomSolidTip() throws Exception{ @Test public void unsolidWithNoRandomTipsAborts() throws Exception{ Mockito.when(tvm.isSolid()).thenReturn(false); - Mockito.when(transactionValidator.quickSetSolid(tvm)).thenReturn(false); + Mockito.when(transactionSolidifier.quickSetSolid(tvm)).thenReturn(false); Mockito.when(tipsViewModel.getRandomSolidTipHash()).thenReturn(null); - SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, transactionValidator, tipsViewModel, tangle); + SolidifyStage solidifyStage = new SolidifyStage(transactionSolidifier, tipsViewModel, tangle); SolidifyPayload solidifyPayload = new SolidifyPayload(null, tvm); ProcessingContext ctx = new ProcessingContext(solidifyPayload); diff --git a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java index 576f418bc0..2e6436353e 100644 --- a/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineTest.java @@ -1,5 +1,7 @@ package com.iota.iri.network.pipeline; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.NodeConfig; @@ -8,7 +10,6 @@ import com.iota.iri.network.SampleTransaction; import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.neighbor.Neighbor; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.storage.Tangle; @@ -43,9 +44,6 @@ public class TransactionProcessingPipelineTest { @Mock private TipsViewModel tipsViewModel; - @Mock - private LatestMilestoneTracker latestMilestoneTracker; - @Mock private TransactionRequester transactionRequester; @@ -70,6 +68,9 @@ public class TransactionProcessingPipelineTest { @Mock private HashingStage hashingStage; + @Mock + private MilestoneStage milestoneStage; + @Mock private ProcessingContext hashingCtx; @@ -85,6 +86,9 @@ public class TransactionProcessingPipelineTest { @Mock private SolidifyPayload solidifyPayload; + @Mock + private MilestonePayload milestonePayload; + @Mock private ProcessingContext validationCtx; @@ -106,6 +110,16 @@ public class TransactionProcessingPipelineTest { @Mock private ProcessingContext abortCtx; + @Mock + private ProcessingContext milestoneCtx; + + @Mock + private MilestoneService milestoneService; + + @Mock + private MilestoneSolidifier milestoneSolidifier; + + @Mock private TransactionSolidifier transactionSolidifier; @@ -125,14 +139,15 @@ private void injectMockedStagesIntoPipeline(TransactionProcessingPipeline pipeli pipeline.setReplyStage(replyStage); pipeline.setValidationStage(validationStage); pipeline.setSolidifyStage(solidifyStage); + pipeline.setMilestoneStage(milestoneStage); } @Test public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, - transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester, transactionSolidifier); + transactionValidator, tangle, snapshotProvider, tipsViewModel, transactionRequester, + transactionSolidifier, milestoneService, milestoneSolidifier); // inject mocks injectMockedStagesIntoPipeline(pipeline); @@ -180,11 +195,68 @@ public void processingAValidNewTransactionFlowsThroughTheEntirePipeline() throws Mockito.verify(broadcastStage).process(Mockito.any()); } + @Test + public void processingAValidMilestone() throws InterruptedException { + TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, + transactionValidator, tangle, snapshotProvider, tipsViewModel, transactionRequester, + transactionSolidifier, milestoneService, milestoneSolidifier); + + injectMockedStagesIntoPipeline(pipeline); + + // mock after pre process context/stage + Mockito.when(preProcessStage.process(Mockito.any())).thenReturn(hashingCtx); + Mockito.when(hashingCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.HASHING); + Mockito.when(hashingCtx.getPayload()).thenReturn(hashingPayload); + + // mock hashing context/stage + mockHashingStage(pipeline); + + // mock validation context/stage + MultiStagePayload divergePayload = new MultiStagePayload(replyCtx, receivedCtx); + Mockito.when(validationStage.process(validationCtx)).thenReturn(divergeToReplyAndReceivedCtx); + Mockito.when(divergeToReplyAndReceivedCtx.getNextStage()) + .thenReturn(TransactionProcessingPipeline.Stage.MULTIPLE); + Mockito.when(divergeToReplyAndReceivedCtx.getPayload()).thenReturn(divergePayload); + + // mock received + Mockito.when(broadcastCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.BROADCAST); + Mockito.when(broadcastCtx.getPayload()).thenReturn(broadcastPayload); + Mockito.when(receivedStage.process(receivedCtx)).thenReturn(milestoneCtx); + + // mock milestone + Mockito.when(milestoneCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.MILESTONE); + Mockito.when(milestoneCtx.getPayload()).thenReturn(milestonePayload); + Mockito.when(milestoneStage.process(milestoneCtx)).thenReturn(solidifyCtx); + + // mock solidify + Mockito.when(solidifyCtx.getPayload()).thenReturn(solidifyPayload); + Mockito.when(solidifyCtx.getNextStage()).thenReturn(TransactionProcessingPipeline.Stage.SOLIDIFY); + Mockito.when(solidifyStage.process(solidifyCtx)).thenReturn(broadcastCtx); + + pipeline.start(); + + // send in actual payload to kick off the 'processing' + pipeline.process(neighbor, SampleTransaction.createSampleTxBuffer()); + + // give it some time to 'process' + Thread.sleep(100); + + // should have called + Mockito.verify(preProcessStage).process(Mockito.any()); + Mockito.verify(hashingStage).process(Mockito.any()); + Mockito.verify(validationStage).process(Mockito.any()); + Mockito.verify(receivedStage).process(Mockito.any()); + Mockito.verify(replyStage).process(Mockito.any()); + Mockito.verify(solidifyStage).process(Mockito.any()); + Mockito.verify(broadcastStage).process(Mockito.any()); + Mockito.verify(milestoneStage).process(Mockito.any()); + } + @Test public void processingAKnownTransactionOnlyFlowsToTheReplyStage() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, - transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester, transactionSolidifier); + transactionValidator, tangle, snapshotProvider, tipsViewModel, transactionRequester, + transactionSolidifier, milestoneService, milestoneSolidifier); // inject mocks pipeline.setPreProcessStage(preProcessStage); @@ -218,8 +290,8 @@ public void processingAKnownTransactionOnlyFlowsToTheReplyStage() throws Interru public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroughTheCorrectStages() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, - transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester, transactionSolidifier); + transactionValidator, tangle, snapshotProvider, tipsViewModel, transactionRequester, + transactionSolidifier, milestoneService, milestoneSolidifier); // inject mocks injectMockedStagesIntoPipeline(pipeline); @@ -268,8 +340,8 @@ public void processingAValidNewTransactionNotOriginatingFromANeighborFlowsThroug @Test public void anInvalidNewTransactionStopsBeingProcessedAfterTheValidationStage() throws InterruptedException { TransactionProcessingPipeline pipeline = new TransactionProcessingPipelineImpl(neighborRouter, nodeConfig, - transactionValidator, tangle, snapshotProvider, tipsViewModel, latestMilestoneTracker, - transactionRequester, transactionSolidifier); + transactionValidator, tangle, snapshotProvider, tipsViewModel, transactionRequester, + transactionSolidifier, milestoneService, milestoneSolidifier); // inject mocks injectMockedStagesIntoPipeline(pipeline); diff --git a/src/test/java/com/iota/iri/service/APITest.java b/src/test/java/com/iota/iri/service/APITest.java index 9643959f54..7169f339ad 100644 --- a/src/test/java/com/iota/iri/service/APITest.java +++ b/src/test/java/com/iota/iri/service/APITest.java @@ -1,5 +1,6 @@ package com.iota.iri.service; +import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.conf.IotaConfig; import com.iota.iri.controllers.TransactionViewModel; @@ -28,6 +29,9 @@ public class APITest { @Mock(answer = Answers.RETURNS_SMART_NULLS) private TransactionValidator transactionValidator; + @Mock + private TransactionSolidifier transactionSolidifier; + @Mock private SnapshotProvider snapshotProvider; @@ -43,7 +47,7 @@ public void whenStoreTransactionsStatementThenSetArrivalTimeToCurrentMillis() th API api = new API(config, null, null, null, null, null, snapshotProvider, null, null, null, null, - transactionValidator, null, null); + transactionValidator, null, null, transactionSolidifier); api.storeTransactionsStatement(Collections.singletonList("FOO")); diff --git a/src/test/java/com/iota/iri/service/ApiCallTest.java b/src/test/java/com/iota/iri/service/ApiCallTest.java index a6f4f0de39..d2ca7dc9b0 100644 --- a/src/test/java/com/iota/iri/service/ApiCallTest.java +++ b/src/test/java/com/iota/iri/service/ApiCallTest.java @@ -14,7 +14,7 @@ public class ApiCallTest { @Before public void setUp() { IotaConfig configuration = Mockito.mock(IotaConfig.class); - api = new API(configuration, null, null, null, null, null, null, null, null, null, null, null, null, null); + api = new API(configuration, null, null, null, null, null, null, null, null, null, null, null, null, null, null); } @Test diff --git a/src/test/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImplTest.java b/src/test/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImplTest.java index 62653ad42f..1f0a7fca58 100644 --- a/src/test/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImplTest.java +++ b/src/test/java/com/iota/iri/service/snapshot/impl/LocalSnapshotManagerImplTest.java @@ -6,6 +6,7 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.any; +import com.iota.iri.service.milestone.MilestoneSolidifier; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -17,7 +18,6 @@ import org.mockito.junit.MockitoRule; import com.iota.iri.conf.SnapshotConfig; -import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotException; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.SnapshotService; @@ -49,7 +49,7 @@ public class LocalSnapshotManagerImplTest { TransactionPruner transactionPruner; @Mock(answer = Answers.RETURNS_DEEP_STUBS) - LatestMilestoneTracker milestoneTracker; + MilestoneSolidifier milestoneSolidifier; private LocalSnapshotManagerImpl lsManager; @@ -71,17 +71,17 @@ public void tearDown() { @Test public synchronized void takeLocalSnapshot() throws SnapshotException { // Always return true - when(milestoneTracker.isInitialScanComplete()).thenReturn(true); + when(milestoneSolidifier.isInitialScanComplete()).thenReturn(true); // When we call it, we are in sync - when(milestoneTracker.getLatestMilestoneIndex()).thenReturn(-5); + when(milestoneSolidifier.getLatestMilestoneIndex()).thenReturn(-5); // We are more then the depth ahead when(snapshotProvider.getLatestSnapshot().getIndex()).thenReturn(100); when(snapshotProvider.getInitialSnapshot().getIndex()).thenReturn(100 - SNAPSHOT_DEPTH - DELAY_SYNC - 1); // Run in separate thread to allow us to time-out - Thread t = new Thread(() -> lsManager.monitorThread(milestoneTracker)); + Thread t = new Thread(() -> lsManager.monitorThread(milestoneSolidifier)); t.start(); // We should finish directly, margin for slower computers @@ -100,40 +100,40 @@ public synchronized void takeLocalSnapshot() throws SnapshotException { @Test public void isInSyncTestScanIncomplete() { - when(milestoneTracker.isInitialScanComplete()).thenReturn(false); + when(milestoneSolidifier.isInitialScanComplete()).thenReturn(false); - assertFalse("We should be out of sync when he havent finished initial scans", lsManager.isInSync(milestoneTracker)); + assertFalse("We should be out of sync when he havent finished initial scans", lsManager.isInSync(milestoneSolidifier)); } @Test public void isInSyncTestScanComplete() { // Always return true - when(milestoneTracker.isInitialScanComplete()).thenReturn(true); + when(milestoneSolidifier.isInitialScanComplete()).thenReturn(true); // We don't really support -1 indexes, but if this breaks, it is a good indication to be careful going further - when(milestoneTracker.getLatestMilestoneIndex()).thenReturn(-1, 5, 10, 998 + BUFFER - 1, 2000); + when(milestoneSolidifier.getLatestMilestoneIndex()).thenReturn(-1, 5, 10, 998 + BUFFER - 1, 2000); // snapshotProvider & milestoneTracker // -5 & -1 -> not in sync - assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneTracker)); + assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneSolidifier)); // -1 and 5 -> not in sync - assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneTracker)); + assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneSolidifier)); // 10 and 10 -> in sync - assertTrue("Equal index should be in sync", lsManager.isInSync(milestoneTracker)); + assertTrue("Equal index should be in sync", lsManager.isInSync(milestoneSolidifier)); // 998 and 1002 -> in sync since sync gap = 5 - assertTrue("Index difference less than the buffer still should be in sync", lsManager.isInSync(milestoneTracker)); + assertTrue("Index difference less than the buffer still should be in sync", lsManager.isInSync(milestoneSolidifier)); // 999 and 2000 -> out of sync again, bigger gap than 5 - assertFalse("Index difference more than the buffer should be out of sync again ", lsManager.isInSync(milestoneTracker)); + assertFalse("Index difference more than the buffer should be out of sync again ", lsManager.isInSync(milestoneSolidifier)); // 1999 and 2000 -> out of sync still - assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneTracker)); + assertFalse("Previous out of sync and not equal index should not be in sync", lsManager.isInSync(milestoneSolidifier)); // 2000 and 2000 -> in sync again - assertTrue("Equal index should be in sync", lsManager.isInSync(milestoneTracker)); + assertTrue("Equal index should be in sync", lsManager.isInSync(milestoneSolidifier)); } @Test diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImplTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImplTest.java index 368aa3bc9b..0ed15f884e 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImplTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImplTest.java @@ -5,7 +5,7 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.IntegerIndex; import com.iota.iri.model.TransactionHash; -import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.impl.SnapshotMockUtils; import com.iota.iri.service.tipselection.EntryPointSelector; @@ -28,7 +28,7 @@ public class EntryPointSelectorImplTest { public MockitoRule mockitoRule = MockitoJUnit.rule(); @Mock - private LatestMilestoneTracker latestMilestoneTracker; + private MilestoneSolidifier milestoneSolidifier; @Mock private Tangle tangle; @@ -50,7 +50,7 @@ public void testEntryPointBWithTangleData() throws Exception { mockTangleBehavior(milestoneHash); mockMilestoneTrackerBehavior(snapshotProvider.getInitialSnapshot().getIndex() + 1, Hash.NULL_HASH); - EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, latestMilestoneTracker); + EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, milestoneSolidifier); Hash entryPoint = entryPointSelector.getEntryPoint(10); Assert.assertEquals("The entry point should be the milestone in the Tangle", milestoneHash, entryPoint); @@ -60,7 +60,7 @@ public void testEntryPointBWithTangleData() throws Exception { public void testEntryPointAWithoutTangleData() throws Exception { mockMilestoneTrackerBehavior(0, Hash.NULL_HASH); - EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, latestMilestoneTracker); + EntryPointSelector entryPointSelector = new EntryPointSelectorImpl(tangle, snapshotProvider, milestoneSolidifier); Hash entryPoint = entryPointSelector.getEntryPoint(10); Assert.assertEquals("The entry point should be the last tracked solid milestone", Hash.NULL_HASH, entryPoint); @@ -70,7 +70,7 @@ public void testEntryPointAWithoutTangleData() throws Exception { private void mockMilestoneTrackerBehavior(int latestSolidSubtangleMilestoneIndex, Hash latestSolidSubtangleMilestone) { snapshotProvider.getLatestSnapshot().setIndex(latestSolidSubtangleMilestoneIndex); snapshotProvider.getLatestSnapshot().setHash(latestSolidSubtangleMilestone); - Mockito.when(latestMilestoneTracker.getLatestMilestoneIndex()).thenReturn(latestSolidSubtangleMilestoneIndex); + Mockito.when(milestoneSolidifier.getLatestMilestoneIndex()).thenReturn(latestSolidSubtangleMilestoneIndex); } private void mockTangleBehavior(Hash milestoneModelHash) throws Exception { diff --git a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java index 873e039159..c8d7d1af3d 100644 --- a/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java +++ b/src/test/java/com/iota/iri/service/validation/impl/TransactionSolidifierImplTest.java @@ -1,5 +1,8 @@ package com.iota.iri.service.validation.impl; +import com.iota.iri.conf.ConfigFactory; +import com.iota.iri.conf.IotaConfig; +import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.crypto.SpongeFactory; import com.iota.iri.model.TransactionHash; @@ -29,6 +32,7 @@ public class TransactionSolidifierImplTest { private static final TemporaryFolder dbFolder = new TemporaryFolder(); private static final TemporaryFolder logFolder = new TemporaryFolder(); private static Tangle tangle; + private IotaConfig config = ConfigFactory.createIotaConfig(true); @Rule public MockitoRule mockitoRule = MockitoJUnit.rule(); @@ -39,6 +43,9 @@ public class TransactionSolidifierImplTest { @Mock private static TransactionSolidifierImpl txSolidifier; + @Mock + private static TipsViewModel tipsViewModel; + @Mock private static TransactionRequester txRequester; @@ -64,7 +71,7 @@ public static void tearDown() throws Exception { public void setUpEach() { when(snapshotProvider.getInitialSnapshot()).thenReturn(SnapshotMockUtils.createSnapshot()); txRequester = new TransactionRequester(tangle, snapshotProvider); - txSolidifier = new TransactionSolidifierImpl(tangle, snapshotProvider, txRequester); + txSolidifier = new TransactionSolidifierImpl(tangle, snapshotProvider, txRequester, tipsViewModel, config.getCoordinator()); txSolidifier.start(); } From c2f8c615c95fabae1cfe80b2974da1f9dcf4ced2 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 30 Dec 2019 17:02:04 -0700 Subject: [PATCH 44/48] Remove old transaction validator --- .../com/iota/iri/TransactionValidator.java | 465 ------------------ 1 file changed, 465 deletions(-) delete mode 100644 src/main/java/com/iota/iri/TransactionValidator.java diff --git a/src/main/java/com/iota/iri/TransactionValidator.java b/src/main/java/com/iota/iri/TransactionValidator.java deleted file mode 100644 index ef51b5c203..0000000000 --- a/src/main/java/com/iota/iri/TransactionValidator.java +++ /dev/null @@ -1,465 +0,0 @@ -package com.iota.iri; - -import com.google.common.annotations.VisibleForTesting; -import com.iota.iri.conf.ProtocolConfig; -import com.iota.iri.controllers.TipsViewModel; -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.crypto.Curl; -import com.iota.iri.crypto.Sponge; -import com.iota.iri.crypto.SpongeFactory; -import com.iota.iri.model.Hash; -import com.iota.iri.model.TransactionHash; -import com.iota.iri.network.TransactionRequester; -import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.storage.Tangle; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.iota.iri.controllers.TransactionViewModel.*; - -public class TransactionValidator { - private static final Logger log = LoggerFactory.getLogger(TransactionValidator.class); - private static final int TESTNET_MWM_CAP = 13; - public static final int SOLID_SLEEP_TIME = 500; - - private final Tangle tangle; - private final SnapshotProvider snapshotProvider; - private final TipsViewModel tipsViewModel; - private final TransactionRequester transactionRequester; - private int minWeightMagnitude = 81; - private static final long MAX_TIMESTAMP_FUTURE = 2L * 60L * 60L; - private static final long MAX_TIMESTAMP_FUTURE_MS = MAX_TIMESTAMP_FUTURE * 1_000L; - - - /////////////////////////////////fields for solidification thread////////////////////////////////////// - - private Thread newSolidThread; - - /** - * If true use {@link #newSolidTransactionsOne} while solidifying. Else use {@link #newSolidTransactionsTwo}. - */ - private final AtomicBoolean useFirst = new AtomicBoolean(true); - /** - * Is {@link #newSolidThread} shutting down - */ - private final AtomicBoolean shuttingDown = new AtomicBoolean(false); - /** - * mutex for solidification - */ - private final Object cascadeSync = new Object(); - private final Set newSolidTransactionsOne = new LinkedHashSet<>(); - private final Set newSolidTransactionsTwo = new LinkedHashSet<>(); - - /** - * Constructor for Tangle Validator - * - * @param tangle relays tangle data to and from the persistence layer - * @param snapshotProvider data provider for the snapshots that are relevant for the node - * @param tipsViewModel container that gets updated with the latest tips (transactions with no children) - * @param transactionRequester used to request missing transactions from neighbors - * @param protocolConfig used for checking if we are in testnet and mwm. testnet true if we are in testnet - * mode, this caps {@code mwm} to {@value #TESTNET_MWM_CAP} regardless of parameter input. - * minimum weight magnitude: the minimal number of 9s that ought to appear at the end of the - * transaction hash - */ - TransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, ProtocolConfig protocolConfig) { - this.tangle = tangle; - this.snapshotProvider = snapshotProvider; - this.tipsViewModel = tipsViewModel; - this.transactionRequester = transactionRequester; - this.newSolidThread = new Thread(spawnSolidTransactionsPropagation(), "Solid TX cascader"); - setMwm(protocolConfig.isTestnet(), protocolConfig.getMwm()); - } - - /** - * Does two things: - *
    - *
  1. Sets the minimum weight magnitude (MWM). POW on a transaction is validated by counting a certain - * number of consecutive 9s in the end of the transaction hash. The number of 9s is the MWM.
    2. - *
  2. Starts the transaction solidification thread.
    3. - *
- * - * - * @see #spawnSolidTransactionsPropagation() - */ - public void init() { - newSolidThread.start(); - } - - @VisibleForTesting - void setMwm(boolean testnet, int mwm) { - minWeightMagnitude = mwm; - - //lowest allowed MWM encoded in 46 bytes. - if (!testnet){ - minWeightMagnitude = Math.max(minWeightMagnitude, TESTNET_MWM_CAP); - } - } - - /** - * Shutdown roots to tip solidification thread - * @throws InterruptedException - * @see #spawnSolidTransactionsPropagation() - */ - public void shutdown() throws InterruptedException { - shuttingDown.set(true); - newSolidThread.join(); - } - - /** - * @return the minimal number of trailing 9s that have to be present at the end of the transaction hash - * in order to validate that sufficient proof of work has been done - */ - public int getMinWeightMagnitude() { - return minWeightMagnitude; - } - - /** - * Checks that the timestamp of the transaction is below the last global snapshot time - * or more than {@value #MAX_TIMESTAMP_FUTURE} seconds in the future, and thus invalid. - * - *

- * First the attachment timestamp (set after performing POW) is checked, and if not available - * the regular timestamp is checked. Genesis transaction will always be valid. - *

- * @param transactionViewModel transaction under test - * @return true if timestamp is not in valid bounds and {@code transactionViewModel} is not genesis. - * Else returns false. - */ - private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) { - // ignore invalid timestamps for transactions that were requested by our node while solidifying a milestone - if(transactionRequester.wasTransactionRecentlyRequested(transactionViewModel.getHash())) { - return false; - } - - if (transactionViewModel.getAttachmentTimestamp() == 0) { - return transactionViewModel.getTimestamp() < snapshotProvider.getInitialSnapshot().getTimestamp() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(transactionViewModel.getHash()) - || transactionViewModel.getTimestamp() > (System.currentTimeMillis() / 1000) + MAX_TIMESTAMP_FUTURE; - } - return transactionViewModel.getAttachmentTimestamp() < (snapshotProvider.getInitialSnapshot().getTimestamp() * 1000L) - || transactionViewModel.getAttachmentTimestamp() > System.currentTimeMillis() + MAX_TIMESTAMP_FUTURE_MS; - } - - /** - * Runs the following validation checks on a transaction: - *
    - *
  1. {@link #hasInvalidTimestamp} check.
    2. - *
  2. Check that no value trits are set beyond the usable index, otherwise we will have values larger - * than max supply.
    3. - *
  3. Check that sufficient POW was performed.
    4. - *
  4. In value transactions, we check that the address has 0 set as the last trit. This must be because of the - * conversion between bytes to trits.
    5. - *
- *Exception is thrown upon failure. - * - * @param transactionViewModel transaction that should be validated - * @param minWeightMagnitude the minimal number of trailing 9s at the end of the transaction hash - * @throws StaleTimestampException if timestamp check fails - * @throws IllegalStateException if any of the other checks fail - */ - public void runValidation(TransactionViewModel transactionViewModel, final int minWeightMagnitude) { - transactionViewModel.setMetadata(); - transactionViewModel.setAttachmentData(); - if(hasInvalidTimestamp(transactionViewModel)) { - throw new StaleTimestampException("Invalid transaction timestamp."); - } - for (int i = VALUE_TRINARY_OFFSET + VALUE_USABLE_TRINARY_SIZE; i < VALUE_TRINARY_OFFSET + VALUE_TRINARY_SIZE; i++) { - if (transactionViewModel.trits()[i] != 0) { - throw new IllegalStateException("Invalid transaction value"); - } - } - - int weightMagnitude = transactionViewModel.weightMagnitude; - if(weightMagnitude < minWeightMagnitude) { - throw new IllegalStateException("Invalid weight magnitude"); - } - - if (transactionViewModel.value() != 0 && transactionViewModel.getAddressHash().trits()[Curl.HASH_LENGTH - 1] != 0) { - throw new IllegalStateException("Invalid transaction address"); - } - } - - /** - * Creates a new transaction from {@code trits} and validates it with {@link #runValidation}. - * - * @param trits raw transaction trits - * @param minWeightMagnitude minimal number of trailing 9s in transaction for POW validation - * @return the transaction resulting from the raw trits if valid. - * @throws RuntimeException if validation fails - */ - public TransactionViewModel validateTrits(final byte[] trits, int minWeightMagnitude) { - TransactionViewModel transactionViewModel = new TransactionViewModel(trits, TransactionHash.calculate(trits, 0, trits.length, SpongeFactory.create(SpongeFactory.Mode.CURLP81))); - runValidation(transactionViewModel, minWeightMagnitude); - return transactionViewModel; - } - - /** - * Creates a new transaction from {@code bytes} and validates it with {@link #runValidation}. - * - * @param bytes raw transaction bytes - * @param minWeightMagnitude minimal number of trailing 9s in transaction for POW validation - * @return the transaction resulting from the raw bytes if valid - * @throws RuntimeException if validation fails - */ - public TransactionViewModel validateBytes(final byte[] bytes, int minWeightMagnitude, Sponge curl) { - TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, TransactionHash.calculate(bytes, TRINARY_SIZE, curl)); - runValidation(transactionViewModel, minWeightMagnitude); - return transactionViewModel; - } - - /** - * This method does the same as {@link #checkSolidity(Hash, int)} but defaults to an unlimited amount - * of transactions that are allowed to be traversed. - * - * @param hash hash of the transactions that shall get checked - * @return true if the transaction is solid and false otherwise - * @throws Exception if anything goes wrong while trying to solidify the transaction - */ - public boolean checkSolidity(Hash hash) throws Exception { - return checkSolidity(hash, Integer.MAX_VALUE); - } - - /** - * This method checks transactions for solidity and marks them accordingly if they are found to be solid. - * - * It iterates through all approved transactions until it finds one that is missing in the database or until it - * reached solid transactions on all traversed subtangles. In case of a missing transactions it issues a transaction - * request and returns false. If no missing transaction is found, it marks the processed transactions as solid in - * the database and returns true. - * - * Since this operation can potentially take a long time to terminate if it would have to traverse big parts of the - * tangle, it is possible to limit the amount of transactions that are allowed to be processed, while looking for - * unsolid / missing approvees. This can be useful when trying to "interrupt" the solidification of one transaction - * (if it takes too many steps) to give another one the chance to be solidified instead (i.e. prevent blocks in the - * solidification threads). - * - * @param hash hash of the transactions that shall get checked - * @param maxProcessedTransactions the maximum amount of transactions that are allowed to be traversed - * @return true if the transaction is solid and false otherwise - * @throws Exception if anything goes wrong while trying to solidify the transaction - */ - public boolean checkSolidity(Hash hash, int maxProcessedTransactions) throws Exception { - if(fromHash(tangle, hash).isSolid()) { - return true; - } - LinkedHashSet analyzedHashes = new LinkedHashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); - if(maxProcessedTransactions != Integer.MAX_VALUE) { - maxProcessedTransactions += analyzedHashes.size(); - } - boolean solid = true; - final Queue nonAnalyzedTransactions = new LinkedList<>(Collections.singleton(hash)); - Hash hashPointer; - while ((hashPointer = nonAnalyzedTransactions.poll()) != null) { - if (!analyzedHashes.add(hashPointer)) { - continue; - } - - if (analyzedHashes.size() >= maxProcessedTransactions) { - return false; - } - - TransactionViewModel transaction = fromHash(tangle, hashPointer); - if (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { - if (transaction.getType() == PREFILLED_SLOT) { - solid = false; - - if (!transactionRequester.isTransactionRequested(hashPointer)) { - transactionRequester.requestTransaction(hashPointer); - continue; - } - } else { - nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); - nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); - } - } - } - if (solid) { - updateSolidTransactions(tangle, snapshotProvider.getInitialSnapshot(), analyzedHashes); - } - analyzedHashes.clear(); - return solid; - } - - public void addSolidTransaction(Hash hash) { - synchronized (cascadeSync) { - if (useFirst.get()) { - newSolidTransactionsOne.add(hash); - } else { - newSolidTransactionsTwo.add(hash); - } - } - } - - /** - * Creates a runnable that runs {@link #propagateSolidTransactions()} in a loop every {@value #SOLID_SLEEP_TIME} ms - * @return runnable that is not started - */ - private Runnable spawnSolidTransactionsPropagation() { - return () -> { - while(!shuttingDown.get()) { - propagateSolidTransactions(); - try { - Thread.sleep(SOLID_SLEEP_TIME); - } catch (InterruptedException e) { - // Ignoring InterruptedException. Do not use Thread.currentThread().interrupt() here. - log.error("Thread was interrupted: ", e); - } - } - }; - } - - /** - * Iterates over all currently known solid transactions. For each solid transaction, we find - * its children (approvers) and try to quickly solidify them with {@link #quietQuickSetSolid}. - * If we manage to solidify the transactions, we add them to the solidification queue for a traversal by a later run. - */ - @VisibleForTesting - void propagateSolidTransactions() { - Set newSolidHashes = new HashSet<>(); - useFirst.set(!useFirst.get()); - //synchronized to make sure no one is changing the newSolidTransactions collections during addAll - synchronized (cascadeSync) { - //We are using a collection that doesn't get updated by other threads - if (useFirst.get()) { - newSolidHashes.addAll(newSolidTransactionsTwo); - newSolidTransactionsTwo.clear(); - } else { - newSolidHashes.addAll(newSolidTransactionsOne); - newSolidTransactionsOne.clear(); - } - } - Iterator cascadeIterator = newSolidHashes.iterator(); - while(cascadeIterator.hasNext() && !shuttingDown.get()) { - try { - Hash hash = cascadeIterator.next(); - TransactionViewModel transaction = fromHash(tangle, hash); - Set approvers = transaction.getApprovers(tangle).getHashes(); - for(Hash h: approvers) { - TransactionViewModel tx = fromHash(tangle, h); - if(quietQuickSetSolid(tx)) { - tx.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); - tipsViewModel.setSolid(h); - addSolidTransaction(h); - } - } - } catch (Exception e) { - log.error("Error while propagating solidity upwards", e); - } - } - } - - - /** - * Updates a transaction after it was stored in the tangle. Tells the node to not request the transaction anymore, - * to update the live tips accordingly, and attempts to quickly solidify the transaction. - * - *

- * Performs the following operations: - * - *

    - *
  1. Removes {@code transactionViewModel}'s hash from the the request queue since we already found it.
    2. - *
  2. If {@code transactionViewModel} has no children (approvers), we add it to the node's active tip list.
    3. - *
  3. Removes {@code transactionViewModel}'s parents (branch & trunk) from the node's tip list - * (if they're present there).
    4. - *
  4. Attempts to quickly solidify {@code transactionViewModel} by checking whether its direct parents - * are solid. If solid we add it to the queue transaction solidification thread to help it propagate the - * solidification to the approving child transactions.
    5. - *
  5. Requests missing direct parent (trunk & branch) transactions that are needed to solidify - * {@code transactionViewModel}.
    6. - *
- * @param transactionViewModel received transaction that is being updated - * @throws Exception if an error occurred while trying to solidify - * @see TipsViewModel - */ - //Not part of the validation process. This should be moved to a component in charge of - //what transaction we gossip. - public void updateStatus(TransactionViewModel transactionViewModel) throws Exception { - transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); - if(transactionViewModel.getApprovers(tangle).size() == 0) { - tipsViewModel.addTipHash(transactionViewModel.getHash()); - } - tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); - tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); - - if(quickSetSolid(transactionViewModel)) { - transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "solid|height"); - tipsViewModel.setSolid(transactionViewModel.getHash()); - addSolidTransaction(transactionViewModel.getHash()); - } - } - - /** - * Perform a {@link #quickSetSolid} while capturing and logging errors - * @param transactionViewModel transaction we try to solidify. - * @return true if we managed to solidify, else false. - */ - private boolean quietQuickSetSolid(TransactionViewModel transactionViewModel) { - try { - return quickSetSolid(transactionViewModel); - } catch (Exception e) { - log.error(e.getMessage(), e); - return false; - } - } - - /** - * Tries to solidify the transactions quickly by performing {@link #checkApproovee} on both parents (trunk and - * branch). If the parents are solid, mark the transactions as solid. - * @param transactionViewModel transaction to solidify - * @return true if we made the transaction solid, else false. - * @throws Exception - */ - private boolean quickSetSolid(final TransactionViewModel transactionViewModel) throws Exception { - if(!transactionViewModel.isSolid()) { - boolean solid = true; - if (!checkApproovee(transactionViewModel.getTrunkTransaction(tangle))) { - solid = false; - } - if (!checkApproovee(transactionViewModel.getBranchTransaction(tangle))) { - solid = false; - } - if(solid) { - transactionViewModel.updateSolid(true); - transactionViewModel.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); - return true; - } - } - return false; - } - - /** - * If the the {@code approvee} is missing, request it from a neighbor. - * @param approovee transaction we check. - * @return true if {@code approvee} is solid. - * @throws Exception if we encounter an error while requesting a transaction - */ - private boolean checkApproovee(TransactionViewModel approovee) throws Exception { - if(snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(approovee.getHash())) { - return true; - } - if(approovee.getType() == PREFILLED_SLOT) { - // don't solidify from the bottom until cuckoo filters can identify where we deleted -> otherwise we will - // continue requesting old transactions forever - //transactionRequester.requestTransaction(approovee.getHash(), false); - return false; - } - return approovee.isSolid(); - } - - @VisibleForTesting - boolean isNewSolidTxSetsEmpty () { - return newSolidTransactionsOne.isEmpty() && newSolidTransactionsTwo.isEmpty(); - } - - /** - * Thrown if transaction fails {@link #hasInvalidTimestamp} check. - */ - public static class StaleTimestampException extends RuntimeException { - StaleTimestampException (String message) { - super(message); - } - } -} From d6b6e0c64507431df1389b9dd5b4a605d25ac4f1 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 30 Dec 2019 15:30:00 -0700 Subject: [PATCH 45/48] Move update status from validation to solidification --- .../java/com/iota/iri/MainInjectionConfiguration.java | 9 ++++----- .../com/iota/iri/network/pipeline/SolidifyStage.java | 3 --- .../pipeline/TransactionProcessingPipelineImpl.java | 1 - src/main/java/com/iota/iri/service/API.java | 6 ++++-- .../iri/service/validation/TransactionValidator.java | 10 +--------- .../iota/iri/network/pipeline/SolidifyStageTest.java | 1 - .../service/validation/TransactionValidatorTest.java | 4 ++-- 7 files changed, 11 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/iota/iri/MainInjectionConfiguration.java b/src/main/java/com/iota/iri/MainInjectionConfiguration.java index 90abd61911..5cf54bf5b9 100644 --- a/src/main/java/com/iota/iri/MainInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/MainInjectionConfiguration.java @@ -135,8 +135,8 @@ LocalSnapshotManager provideLocalSnapshotManager(SnapshotProvider snapshotProvid @Singleton @Provides - TransactionValidator provideTransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester) { - return new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester, configuration); + TransactionValidator provideTransactionValidator(SnapshotProvider snapshotProvider, TransactionRequester transactionRequester) { + return new TransactionValidator(snapshotProvider, transactionRequester, configuration); } @Singleton @@ -173,11 +173,10 @@ IXI provideIxi(Iota iota) { @Singleton @Provides API provideApi(IXI ixi, TransactionRequester transactionRequester, - SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, + SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, SnapshotProvider snapshotProvider, LedgerService ledgerService, NeighborRouter neighborRouter, TipSelector tipsSelector, TipsViewModel tipsViewModel, TransactionValidator transactionValidator, - MilestoneSolidifier milestoneSolidifier, TransactionProcessingPipeline txPipeline, - TransactionSolidifier transactionSolidifier) { + TransactionProcessingPipeline txPipeline, TransactionSolidifier transactionSolidifier, MilestoneSolidifier milestoneSolidifier) { return new API(configuration, ixi, transactionRequester, spentAddressesService, tangle, bundleValidator, snapshotProvider, ledgerService, neighborRouter, tipsSelector, tipsViewModel, transactionValidator, milestoneSolidifier, txPipeline, transactionSolidifier); } diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java index 6fcadecda3..8fbcf921bb 100644 --- a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java @@ -8,7 +8,6 @@ import com.iota.iri.network.neighbor.Neighbor; import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.service.validation.TransactionSolidifier; -import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Tangle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,11 +26,9 @@ public class SolidifyStage implements Stage { private static final Logger log = LoggerFactory.getLogger(SolidifyStage.class); private TransactionSolidifier txSolidifier; - private TransactionValidator txValidator; private TipsViewModel tipsViewModel; private Tangle tangle; private TransactionViewModel tip; - private MilestoneService milestoneService; /** * Constructor for the {@link SolidifyStage}. diff --git a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java index 2a60f20488..7de9eadb55 100644 --- a/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java +++ b/src/main/java/com/iota/iri/network/pipeline/TransactionProcessingPipelineImpl.java @@ -26,7 +26,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 86057dc72a..3edefc041f 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -9,6 +9,7 @@ import com.iota.iri.service.milestone.MilestoneSolidifier; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.service.validation.TransactionValidator; +import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.conf.APIConfig; import com.iota.iri.conf.IotaConfig; import com.iota.iri.controllers.*; @@ -111,7 +112,7 @@ public class API { private final TransactionValidator transactionValidator; private final MilestoneSolidifier milestoneSolidifier; private final TransactionSolidifier transactionSolidifier; - + private final int maxFindTxs; private final int maxRequestList; private final int maxGetTrytes; @@ -151,6 +152,7 @@ public class API { * @param tipsViewModel Contains the current tips of this node * @param transactionValidator Validates transactions * @param milestoneSolidifier Service that tracks the latest milestone + * @param transactionSolidifier Holds transaction pipeline, including broadcast transactions */ public API(IotaConfig configuration, IXI ixi, TransactionRequester transactionRequester, SpentAddressesService spentAddressesService, Tangle tangle, BundleValidator bundleValidator, @@ -174,7 +176,7 @@ public API(IotaConfig configuration, IXI ixi, TransactionRequester transactionRe this.transactionValidator = transactionValidator; this.milestoneSolidifier = milestoneSolidifier; this.transactionSolidifier = transactionSolidifier; - + maxFindTxs = configuration.getMaxFindTransactions(); maxRequestList = configuration.getMaxRequestsList(); maxGetTrytes = configuration.getMaxGetTrytes(); diff --git a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java index 3350ecbaa7..f2177b585f 100644 --- a/src/main/java/com/iota/iri/service/validation/TransactionValidator.java +++ b/src/main/java/com/iota/iri/service/validation/TransactionValidator.java @@ -2,7 +2,6 @@ import com.google.common.annotations.VisibleForTesting; import com.iota.iri.conf.ProtocolConfig; -import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.crypto.Curl; import com.iota.iri.crypto.Sponge; @@ -10,16 +9,13 @@ import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.storage.Tangle; import static com.iota.iri.controllers.TransactionViewModel.*; public class TransactionValidator { private static final int TESTNET_MWM_CAP = 13; - private final Tangle tangle; private final SnapshotProvider snapshotProvider; - private final TipsViewModel tipsViewModel; private final TransactionRequester transactionRequester; private int minWeightMagnitude = 81; private static final long MAX_TIMESTAMP_FUTURE = 2L * 60L * 60L; @@ -29,19 +25,15 @@ public class TransactionValidator { /** * Constructor for Tangle Validator * - * @param tangle relays tangle data to and from the persistence layer * @param snapshotProvider data provider for the snapshots that are relevant for the node - * @param tipsViewModel container that gets updated with the latest tips (transactions with no children) * @param transactionRequester used to request missing transactions from neighbors * @param protocolConfig used for checking if we are in testnet and mwm. testnet true if we are in testnet * mode, this caps {@code mwm} to {@value #TESTNET_MWM_CAP} regardless of parameter input. * minimum weight magnitude: the minimal number of 9s that ought to appear at the end of the * transaction hash */ - public TransactionValidator(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, TransactionRequester transactionRequester, ProtocolConfig protocolConfig) { - this.tangle = tangle; + public TransactionValidator(SnapshotProvider snapshotProvider, TransactionRequester transactionRequester, ProtocolConfig protocolConfig) { this.snapshotProvider = snapshotProvider; - this.tipsViewModel = tipsViewModel; this.transactionRequester = transactionRequester; setMwm(protocolConfig.isTestnet(), protocolConfig.getMwm()); } diff --git a/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java b/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java index e79e27ccce..958c5f3259 100644 --- a/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java +++ b/src/test/java/com/iota/iri/network/pipeline/SolidifyStageTest.java @@ -5,7 +5,6 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.persistables.Transaction; import com.iota.iri.service.validation.TransactionSolidifier; -import com.iota.iri.service.validation.TransactionValidator; import com.iota.iri.storage.Tangle; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java b/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java index 8ef6ab2245..d27ca7164c 100644 --- a/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/service/validation/TransactionValidatorTest.java @@ -66,7 +66,7 @@ public void setUpEach() { when(snapshotProvider.getInitialSnapshot()).thenReturn(SnapshotMockUtils.createSnapshot()); TipsViewModel tipsViewModel = new TipsViewModel(); txRequester = new TransactionRequester(tangle, snapshotProvider); - txValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, txRequester, new MainnetConfig()); + txValidator = new TransactionValidator(snapshotProvider, txRequester, new MainnetConfig()); txValidator.setMwm(false, MAINNET_MWM); } @@ -74,7 +74,7 @@ public void setUpEach() { public void testMinMwm() { ProtocolConfig protocolConfig = mock(ProtocolConfig.class); when(protocolConfig.getMwm()).thenReturn(5); - TransactionValidator transactionValidator = new TransactionValidator(null, null, null, null, protocolConfig); + TransactionValidator transactionValidator = new TransactionValidator(null, null, protocolConfig); assertEquals("Expected testnet minimum minWeightMagnitude", 13, transactionValidator.getMinWeightMagnitude()); } From 503836d643c6b91513808cdeb60e5ba6e974f4e3 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 3 Feb 2020 13:18:23 -0700 Subject: [PATCH 46/48] load -> get --- .../iri/service/milestone/impl/MilestoneSolidifierImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java index 74a8236472..fc7320535d 100644 --- a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java @@ -212,7 +212,7 @@ private void bootStrapSolidMilestones() throws Exception { boolean solid = true; int milestoneIndex = snapshotProvider.getInitialSnapshot().getIndex(); while(solid){ - if(!MilestoneViewModel.load(tangle, milestoneIndex)){ + if(MilestoneViewModel.get(tangle, milestoneIndex) == null){ solid = false; } milestoneIndex += 1; From 61c42c7d380eaad5d7cd70c71ba6ed367d9d85ce Mon Sep 17 00:00:00 2001 From: DyrellC Date: Mon, 3 Feb 2020 15:04:22 -0700 Subject: [PATCH 47/48] update bootstrap logic --- .../iri/service/milestone/impl/MilestoneSolidifierImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java index fc7320535d..cd13ee7d67 100644 --- a/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java +++ b/src/main/java/com/iota/iri/service/milestone/impl/MilestoneSolidifierImpl.java @@ -212,10 +212,11 @@ private void bootStrapSolidMilestones() throws Exception { boolean solid = true; int milestoneIndex = snapshotProvider.getInitialSnapshot().getIndex(); while(solid){ + milestoneIndex += 1; if(MilestoneViewModel.get(tangle, milestoneIndex) == null){ solid = false; + milestoneIndex -= 1; } - milestoneIndex += 1; } latestSolidMilestone = milestoneIndex; From 6ae83076dc0635dc0230ab4045a235e7e728c8c8 Mon Sep 17 00:00:00 2001 From: DyrellC Date: Fri, 14 Feb 2020 18:41:52 -0700 Subject: [PATCH 48/48] Fix solidify -> broadcast --- .../iri/network/pipeline/SolidifyStage.java | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java index 8fbcf921bb..49f285a509 100644 --- a/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java +++ b/src/main/java/com/iota/iri/network/pipeline/SolidifyStage.java @@ -4,9 +4,6 @@ import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.model.persistables.Transaction; -import com.iota.iri.network.neighbor.Neighbor; -import com.iota.iri.service.milestone.MilestoneService; import com.iota.iri.service.validation.TransactionSolidifier; import com.iota.iri.storage.Tangle; import org.slf4j.Logger; @@ -33,7 +30,7 @@ public class SolidifyStage implements Stage { /** * Constructor for the {@link SolidifyStage}. * - * @param txSolidifier Transaction solidifier implementation for updating transaction's solidity status + * @param txSolidifier Transaction validator implementation for determining the validity of a transaction * @param tipsViewModel Used for broadcasting random solid tips if the subject transaction is unsolid * @param tangle A reference to the nodes DB */ @@ -46,9 +43,8 @@ public SolidifyStage(TransactionSolidifier txSolidifier, TipsViewModel tipsViewM /** * Processes the payload of the {@link ProcessingContext} as a {@link SolidifyPayload}. First the transaction will * be checked for solidity and validity. If the transaction is already solid or can be set solid quickly by the - * transaction validator, the transaction is passed to the {@link BroadcastStage}. If not, the transaction is - * added to the solidification queue, and a random solid tip is pulled form the {@link TipsViewModel} to be - * broadcast instead. + * transaction validator, the transaction is passed to the {@link BroadcastStage}. If not, a random solid tip is + * pulled form the {@link TipsViewModel} to be broadcast instead. * * @param ctx The context to process * @return The output context, in most cases a {@link BroadcastPayload}. @@ -60,10 +56,10 @@ public ProcessingContext process(ProcessingContext ctx){ TransactionViewModel tvm = payload.getTransaction(); if (tvm.isSolid() || txSolidifier.quickSetSolid(tvm)) { - return prepareBroadcast(ctx, payload.getOriginNeighbor(), tvm); + tip = tvm; } - return broadcastTip(ctx, payload, tvm); + return broadcastTip(ctx, payload); }catch (Exception e){ log.error("Failed to process transaction for solidification", e); ctx.setNextStage(TransactionProcessingPipeline.Stage.ABORT); @@ -72,8 +68,7 @@ public ProcessingContext process(ProcessingContext ctx){ } - private ProcessingContext broadcastTip(ProcessingContext ctx, SolidifyPayload payload, - TransactionViewModel tvm) throws Exception{ + private ProcessingContext broadcastTip(ProcessingContext ctx, SolidifyPayload payload) throws Exception{ if(tip == null) { Hash tipHash = tipsViewModel.getRandomSolidTipHash(); @@ -85,16 +80,9 @@ private ProcessingContext broadcastTip(ProcessingContext ctx, SolidifyPayload pa tip = fromHash(tangle, tipHash); } - if(tip.isSolid()) { - return prepareBroadcast(ctx, payload.getOriginNeighbor(), tip); - } else { - return prepareBroadcast(ctx, null, tvm); - } - } - - private ProcessingContext prepareBroadcast(ProcessingContext ctx, Neighbor originNeighbor, TransactionViewModel tx){ ctx.setNextStage(TransactionProcessingPipeline.Stage.BROADCAST); - ctx.setPayload(new BroadcastPayload(originNeighbor, tx)); + ctx.setPayload(new BroadcastPayload(payload.getOriginNeighbor(), tip)); + tip = null; return ctx; }