diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 48ed6875fe..df8540cae7 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -21,8 +21,12 @@ import com.iota.iri.service.spentaddresses.impl.SpentAddressesServiceImpl; import com.iota.iri.service.tipselection.*; import com.iota.iri.service.tipselection.impl.*; +import com.iota.iri.service.transactionpruning.PrunedTransactionException; +import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionVerifierImpl; import com.iota.iri.storage.*; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Pair; @@ -95,6 +99,10 @@ public class Iota { public final TransactionRequesterWorkerImpl transactionRequesterWorker; + public final PrunedTransactionProviderImpl prunedTransactionProvider; + + public final PrunedTransactionVerifier prunedTransactionVerifier; + public final BundleValidator bundleValidator; public final Tangle tangle; @@ -108,6 +116,8 @@ public class Iota { public final TipsViewModel tipsViewModel; public final TipSelector tipsSelector; + + /** * Initializes the latest snapshot and then creates all services needed to run an IOTA node. * @@ -115,8 +125,9 @@ public class Iota { * @throws TransactionPruningException If the TransactionPruner could not restore its state. * @throws SnapshotException If the Snapshot fails to initialize. * This can happen if the snapshot signature is invalid or the file cannot be read. + * @throws PrunedTransactionException If we could not load previously pruned transactions (whilst they exist) */ - public Iota(IotaConfig configuration) throws TransactionPruningException, SnapshotException, SpentAddressesException { + public Iota(IotaConfig configuration) throws TransactionPruningException, SnapshotException, SpentAddressesException, PrunedTransactionException { this.configuration = configuration; // new refactored instances @@ -135,6 +146,7 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh transactionPruner = configuration.getLocalSnapshotsEnabled() && configuration.getLocalSnapshotsPruningEnabled() ? new AsyncTransactionPruner() : null; + transactionRequesterWorker = new TransactionRequesterWorkerImpl(); // legacy code @@ -143,8 +155,13 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh tipsViewModel = new TipsViewModel(); transactionRequester = new TransactionRequester(tangle, snapshotProvider); transactionValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester); + + prunedTransactionProvider = transactionPruner != null ? new PrunedTransactionProviderImpl() : null; + prunedTransactionVerifier = transactionPruner != null ? new PrunedTransactionVerifierImpl( + prunedTransactionProvider, transactionRequester) : null; + node = new Node(tangle, snapshotProvider, transactionValidator, transactionRequester, tipsViewModel, - latestMilestoneTracker, configuration); + latestMilestoneTracker, configuration, prunedTransactionVerifier); replicator = new Replicator(node, configuration); udpReceiver = new UDPReceiver(node, configuration); tipsSolidifier = new TipsSolidifier(tangle, transactionValidator, tipsViewModel, configuration); @@ -175,7 +192,7 @@ public void init() throws Exception { tangle.clearMetadata(com.iota.iri.model.persistables.Transaction.class); } - transactionValidator.init(configuration.isTestnet(), configuration.getMwm()); + transactionValidator.init(configuration.isTestnet(), configuration.getMwm(), prunedTransactionVerifier); tipsSolidifier.init(); transactionRequester.init(configuration.getpRemoveRequest()); udpReceiver.init(); @@ -196,7 +213,9 @@ public void init() throws Exception { } } - private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException { + private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException, + PrunedTransactionException { + //snapshot provider must be initialized first //because we check whether spent addresses data exists snapshotProvider.init(configuration); @@ -216,8 +235,14 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, spentAddressesService, bundleValidator); if (transactionPruner != null) { - transactionPruner.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, tipsViewModel, configuration); + if (prunedTransactionProvider != null) { + prunedTransactionProvider.init(configuration); + } + transactionPruner.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, + tipsViewModel, configuration, prunedTransactionProvider); } + + transactionRequesterWorker.init(tangle, transactionRequester, tipsViewModel, node); } diff --git a/src/main/java/com/iota/iri/TransactionValidator.java b/src/main/java/com/iota/iri/TransactionValidator.java index 022e5c5d23..a797128cdf 100644 --- a/src/main/java/com/iota/iri/TransactionValidator.java +++ b/src/main/java/com/iota/iri/TransactionValidator.java @@ -10,6 +10,7 @@ import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier; import com.iota.iri.storage.Tangle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +53,8 @@ public class TransactionValidator { private final Set newSolidTransactionsOne = new LinkedHashSet<>(); private final Set newSolidTransactionsTwo = new LinkedHashSet<>(); + private PrunedTransactionVerifier prunedTransactionVerifier; + /** * Constructor for Tangle Validator * @@ -81,9 +84,12 @@ public class TransactionValidator { * regardless of parameter input. * @param mwm minimum weight magnitude: the minimal number of 9s that ought to appear at the end of the transaction * hash + * @param prunedTransactionProviderProvider for checking if a transaction is pruned previously */ - public void init(boolean testnet, int mwm) { + public void init(boolean testnet, int mwm, PrunedTransactionVerifier prunedTransactionVerifier) { setMwm(testnet, mwm); + + this.prunedTransactionVerifier = prunedTransactionVerifier; newSolidThread = new Thread(spawnSolidTransactionsPropagation(), "Solid TX cascader"); newSolidThread.start(); @@ -247,6 +253,13 @@ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTrans if(fromHash(tangle, hash).isSolid()) { return true; } + + // isPruned gets updated through parent CF verification. + // If one check is negative, isPossiblyPruned will return false. + if (this.prunedTransactionVerifier != null && this.prunedTransactionVerifier.isPossiblyPruned(hash)){ + return this.prunedTransactionVerifier.isPruned(hash); + } + Set analyzedHashes = new HashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); if(maxProcessedTransactions != Integer.MAX_VALUE) { maxProcessedTransactions += analyzedHashes.size(); diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index 1c2e2288e1..c203e547e1 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -5,7 +5,6 @@ import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.iota.iri.IRI; import com.iota.iri.crypto.SpongeFactory; import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; @@ -111,7 +110,9 @@ public abstract class BaseIotaConfig implements IotaConfig { protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH; protected String spentAddressesDbPath = Defaults.SPENT_ADDRESSES_DB_PATH; protected String spentAddressesDbLogPath = Defaults.SPENT_ADDRESSES_DB_LOG_PATH; - + protected String prunedTransactionsDbPath = Defaults.PRUNED_TRANSACTIONS_DB_PATH; + protected String prunedTransactionsDbLogPath = Defaults.PRUNED_TRANSACTIONS_DB_LOG_PATH; + public BaseIotaConfig() { //empty constructor } @@ -667,13 +668,35 @@ protected void setSpentAddressesDbPath(String spentAddressesDbPath) { public String getSpentAddressesDbLogPath() { return spentAddressesDbLogPath; } - + @JsonProperty @Parameter(names = {"--spent-addresses-db-log-path"}, description = SnapshotConfig.Descriptions.SPENT_ADDRESSES_DB_LOG_PATH) protected void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) { this.spentAddressesDbLogPath = spentAddressesDbLogPath; } - + + @Override + public String getPrunedTransactionsDbLogPath() { + return prunedTransactionsDbLogPath; + } + + @JsonProperty + @Parameter(names = {"--pruned-transactions-db-log-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_LOG_PATH) + protected void setPrunedTransactionDbLogPath(String prunedTransactionsDbLogPath) { + this.prunedTransactionsDbLogPath = prunedTransactionsDbLogPath; + } + + @Override + public String getPrunedTransactionsDbPath() { + return prunedTransactionsDbPath; + } + + @JsonProperty + @Parameter(names = {"--pruned-transactions-db-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_PATH) + protected void setPrunedTransactionsDbPath(String prunedTransactionsDbPath) { + this.prunedTransactionsDbPath = prunedTransactionsDbPath; + } + /** * Checks if ZMQ is enabled. * @return true if zmqEnableTcp or zmqEnableIpc is set. @@ -951,6 +974,9 @@ public interface Defaults { int LOCAL_SNAPSHOTS_DEPTH_MIN = 100; String SPENT_ADDRESSES_DB_PATH = "spent-addresses-db"; String SPENT_ADDRESSES_DB_LOG_PATH = "spent-addresses-log"; + + String PRUNED_TRANSACTIONS_DB_LOG_PATH = "spent-addresses-db"; + String PRUNED_TRANSACTIONS_DB_PATH = "spent-addresses-log"; String LOCAL_SNAPSHOTS_BASE_PATH = "mainnet"; String SNAPSHOT_FILE = "/snapshotMainnet.txt"; diff --git a/src/main/java/com/iota/iri/conf/SnapshotConfig.java b/src/main/java/com/iota/iri/conf/SnapshotConfig.java index 33c30fb705..b213c4f8b6 100644 --- a/src/main/java/com/iota/iri/conf/SnapshotConfig.java +++ b/src/main/java/com/iota/iri/conf/SnapshotConfig.java @@ -76,6 +76,16 @@ public interface SnapshotConfig extends Config { */ String getSpentAddressesDbLogPath(); + /** + * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_PATH} + */ + String getPrunedTransactionsDbPath(); + + /** + * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_LOG_PATH} + */ + String getPrunedTransactionsDbLogPath(); + interface Descriptions { String LOCAL_SNAPSHOTS_ENABLED = "Flag that determines if local snapshots are enabled."; @@ -94,5 +104,8 @@ interface Descriptions { "from previous epochs"; String SPENT_ADDRESSES_DB_PATH = "The folder where the spent addresses DB saves its data."; String SPENT_ADDRESSES_DB_LOG_PATH = "The folder where the spent addresses DB saves its logs."; + String PRUNED_TRANSACTIONS_DB_PATH = "The folder where the pruned transactions DB saves its data."; + String PRUNED_TRANSACTIONS_DB_LOG_PATH = "The folder where the pruned transactions DB saves its logs."; } + } diff --git a/src/main/java/com/iota/iri/model/persistables/Cuckoo.java b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java new file mode 100644 index 0000000000..6e397c759a --- /dev/null +++ b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java @@ -0,0 +1,82 @@ +package com.iota.iri.model.persistables; + +import java.util.BitSet; + +import org.apache.commons.lang3.ArrayUtils; + +import com.iota.iri.model.IntegerIndex; +import com.iota.iri.storage.Persistable; +import com.iota.iri.utils.Serializer; + +/** + * Persistable to manage the data we get as a result of pruning a milestone and its transactions + */ +public class Cuckoo implements Persistable { + + /** + * The filter number it belonged to in previous cycles + */ + public IntegerIndex filterId; + + /** + * The bits that make up the CF + */ + public BitSet filterBits; + + /** + * + * {@inheritDoc} + */ + @Override + public byte[] bytes() { + byte[] num = filterId.bytes(); + return ArrayUtils.addAll(num, filterBits.toByteArray()); + } + + /** + * Reads a CuckooBucket from the provided bytes. + * First 4 bytes are the bucket id, second 4 are the index inside that bucket + * Rest of the bytes is the bucket data + * + * {@inheritDoc} + */ + @Override + public void read(byte[] bytes) { + if(bytes != null) { + filterId = new IntegerIndex(Serializer.getInteger(bytes, 0)); + + short start = 4; + filterBits = new BitSet(bytes.length - start); + for (int i = start; i < bytes.length; i++) { + filterBits.set(i-start, bytes[i]); + } + } + } + + /** + * + * {@inheritDoc} + */ + @Override + public byte[] metadata() { + return new byte[0]; + } + + /** + * + * {@inheritDoc} + */ + @Override + public void readMetadata(byte[] bytes) { + } + + /** + * + * {@inheritDoc} + */ + @Override + public boolean merge() { + return false; + } + +} diff --git a/src/main/java/com/iota/iri/network/Node.java b/src/main/java/com/iota/iri/network/Node.java index 9e72d545c1..3f9f760ee2 100644 --- a/src/main/java/com/iota/iri/network/Node.java +++ b/src/main/java/com/iota/iri/network/Node.java @@ -10,6 +10,8 @@ import com.iota.iri.model.TransactionHash; import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.transactionpruning.PrunedTransactionException; +import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier; import com.iota.iri.storage.Tangle; import net.openhft.hashing.LongHashFunction; import org.apache.commons.lang3.StringUtils; @@ -69,7 +71,8 @@ public class Node { private final TransactionValidator transactionValidator; private final LatestMilestoneTracker latestMilestoneTracker; private final TransactionRequester transactionRequester; - + private final PrunedTransactionVerifier prunedTransactionVerifier; + private static final SecureRandom rnd = new SecureRandom(); @@ -99,7 +102,10 @@ public class Node { * @param configuration Contains all the config. * */ - public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, final LatestMilestoneTracker latestMilestoneTracker, final NodeConfig configuration + public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, + final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, + final LatestMilestoneTracker latestMilestoneTracker, final NodeConfig configuration, + final PrunedTransactionVerifier prunedTransactionVerifier ) { this.configuration = configuration; this.tangle = tangle; @@ -112,6 +118,7 @@ public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final Transa int packetSize = configuration.getTransactionPacketSize(); this.sendingPacket = new DatagramPacket(new byte[packetSize], packetSize); this.tipRequestingPacket = new DatagramPacket(new byte[packetSize], packetSize); + this.prunedTransactionVerifier = prunedTransactionVerifier; } @@ -445,6 +452,20 @@ public void replyToRequestFromQueue() { public void processReceivedData(TransactionViewModel receivedTransactionViewModel, Neighbor neighbor) { boolean stored = false; + + try { + if (prunedTransactionVerifier != null && + prunedTransactionVerifier.waitingForHash(receivedTransactionViewModel.getHash())) { + + prunedTransactionVerifier.submitTransaction(receivedTransactionViewModel); + + // We dont store these old/pruned transactions + return; + } + } catch (PrunedTransactionException e) { + // We could not verify if this was pruned or not. Handle like normal + log.warn("Failed checking for pruned transaction state.", e); + } //store new transaction try { diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index c59c61dc6e..8bcc7503c7 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -54,8 +54,7 @@ public SpentAddressesProviderImpl init(SnapshotConfig config) {{put("spent-addresses", SpentAddress.class);}}, null); this.rocksDBPersistenceProvider.init(); readPreviousEpochsSpentAddresses(); - } - catch (Exception e) { + } catch (Exception e) { throw new SpentAddressesException("There is a problem with accessing stored spent addresses", e); } return this; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java new file mode 100644 index 0000000000..4e1ea756da --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java @@ -0,0 +1,38 @@ +package com.iota.iri.service.transactionpruning; + +/** + * This class is used to wrap exceptions that are specific to the pruned transaction persistance. + * + * It allows us to distinct between the different kinds of errors that can happen during the execution of the code. + */ +public class PrunedTransactionException extends Exception { + /** + * Constructor of the exception which allows us to provide a specific error message and the cause of the error. + * + * @param message reason why this error occurred + * @param cause wrapped exception that caused this error + */ + public PrunedTransactionException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor of the exception which allows us to provide a specific error message without having an underlying + * cause. + * + * @param message reason why this error occurred + */ + public PrunedTransactionException(String message) { + super(message); + } + + /** + * Constructor of the exception which allows us to wrap the underlying cause of the error without providing a + * specific reason. + * + * @param cause wrapped exception that caused this error + */ + public PrunedTransactionException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java new file mode 100644 index 0000000000..ea449b2f22 --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java @@ -0,0 +1,36 @@ +package com.iota.iri.service.transactionpruning; + +import java.util.Collection; + +import com.iota.iri.model.Hash; + +/** + * Find, mark and store pruned transactions + */ +public interface PrunedTransactionProvider { + + /** + * Checks if this transactions has been pruned + * + * @param transactionHash The transaction to check for + * @return true if it is, else false + * @throws PrunedTransactionException If the provider fails to check the transaction + */ + boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException; + + /** + * Mark a transaction as spent. + * + * @param transactionHash the transaction which we want to mark. + * @throws PrunedTransactionException If the provider fails to add the transaction + */ + void addTransaction(Hash transactionHash) throws PrunedTransactionException; + + /** + * Mark all transactions as pruned. + * + * @param transactionHashes The transactions we want to mark + * @throws PrunedTransactionException If the provider fails to add a transaction + */ + void addTransactionBatch(Collection transactionHashes) throws PrunedTransactionException; +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java new file mode 100644 index 0000000000..4cef983a2a --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java @@ -0,0 +1,48 @@ +package com.iota.iri.service.transactionpruning; + +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; + +/** + * Verifies the integrity of a chain of pruned transactions by checking each trunk/branch. + */ +public interface PrunedTransactionVerifier { + + /** + * Does a preliminary check to see if we should continue checking this hash its pruned status + * + * @param hash + * @return + * @throws PrunedTransactionException + */ + boolean isPossiblyPruned(Hash hash) throws PrunedTransactionException; + + /** + * Performs multiple checks on the hash to ensure it is pruned before + * + * @param hash + * @return + * @throws PrunedTransactionException + */ + boolean isPruned(Hash hash) throws PrunedTransactionException; + + /** + * Sends the transaction data for the verifier to use/store as it needs. + * {@link #waitingForHash(Hash)} should be used before to determine if we actually need it + * + * @param receivedTransactionViewModel + * @throws PrunedTransactionException + */ + void submitTransaction(TransactionViewModel receivedTransactionViewModel) throws PrunedTransactionException; + + /** + * Checks if we are waiting for this transaction its information. + * THis would mean that {@link #isPruned(Hash)} is called, and one of its parents has a reference to this hash. + * + * @param hash + * @return + * @throws PrunedTransactionException + */ + boolean waitingForHash(Hash hash) throws PrunedTransactionException; + +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java index 619b59a5e9..5923cd5bab 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java @@ -121,6 +121,20 @@ public interface TransactionPrunerJob { * @param transactionPrunerJobStatus new execution status of the job */ void setStatus(TransactionPrunerJobStatus transactionPrunerJobStatus); + + /** + * Getter for the pruned transaction maintainer. + * + * @return pruned transaction maintainer of the job. + */ + PrunedTransactionProvider getPrunedProvider(); + + /** + * Setter for the pruned transaction maintainer. + * + * @param prunedTransactionProvider pruned transaction maintainer of the job + */ + void setPrunedProvider(PrunedTransactionProvider prunedTransactionProvider); /** * This method processes the cleanup job and performs the actual pruning. diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index 4d9e0632a3..9c19afee05 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -5,9 +5,11 @@ import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPruningException; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; import com.iota.iri.service.transactionpruning.jobs.MilestonePrunerJob; import com.iota.iri.service.transactionpruning.jobs.UnconfirmedSubtanglePrunerJob; import com.iota.iri.storage.Tangle; @@ -110,6 +112,11 @@ public class AsyncTransactionPruner implements TransactionPruner { */ private final Map, JobQueue> jobQueues = new HashMap<>(); + /** + * Provider for managing transactions we delete from the database in an optimized data structure + */ + private PrunedTransactionProvider prunedTransactionProvider; + /** * This method initializes the instance and registers its dependencies.
*
@@ -126,18 +133,21 @@ public class AsyncTransactionPruner implements TransactionPruner { * @param snapshotProvider data provider for the snapshots that are relevant for the node * @param tipsViewModel manager for the tips (required for removing pruned transactions from this manager) * @param config Configuration with important snapshot related configuration parameters + * @param prunedTransactionProvider manager which has logic about how to handle pruned transactions * @return the initialized instance itself to allow chaining */ public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvider, SpentAddressesService spentAddressesService, SpentAddressesProvider spentAddressesProvider, TipsViewModel tipsViewModel, - SnapshotConfig config) { + SnapshotConfig config, + PrunedTransactionProviderImpl prunedTransactionProvider) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.spentAddressesService = spentAddressesService; this.spentAddressesProvider = spentAddressesProvider; + this.prunedTransactionProvider = prunedTransactionProvider; this.tipsViewModel = tipsViewModel; this.config = config; @@ -158,6 +168,7 @@ public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvi @Override public void addJob(TransactionPrunerJob job) throws TransactionPruningException { job.setTransactionPruner(this); + job.setPrunedProvider(prunedTransactionProvider); job.setSpentAddressesService(spentAddressesService); job.setSpentAddressesProvider(spentAddressesProvider); job.setTangle(tangle); diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java new file mode 100644 index 0000000000..afc197acd9 --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -0,0 +1,260 @@ +package com.iota.iri.service.transactionpruning.impl; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.commons.collections4.queue.CircularFifoQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.model.Hash; +import com.iota.iri.model.IntegerIndex; +import com.iota.iri.model.persistables.Cuckoo; +import com.iota.iri.service.spentaddresses.SpentAddressesException; +import com.iota.iri.service.transactionpruning.PrunedTransactionException; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; +import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; +import com.iota.iri.utils.datastructure.CuckooFilter; +import com.iota.iri.utils.datastructure.impl.CuckooFilterImpl; + +/** + * Implementation of a pruned transaction provider which uses a cuckoo filter to store pruned hashes + */ +public class PrunedTransactionProviderImpl implements PrunedTransactionProvider { + + private static final Logger log = LoggerFactory.getLogger(PrunedTransactionProviderImpl.class); + + /** + * The amount of fingerprints each bucket keeps. + * More than 4 will be slower in performance, whilst the extra space it gets us is not needed. + */ + private static final int BUCKET_SIZE = 4; + + /** + * The amount of bits a hash gets transformed to (Higher is better, but uses more storage) + */ + private static final int FINGER_PRINT_SIZE = 16; + + /** + * The estimated amount of fingerprints each filter should hold (Will be scaled by cuckoo impl.) + */ + private static final int FILTER_SIZE = 200000; + + /** + * The maximum amount of filters we have in use at any given time + */ + private static final int MAX_FILTERS = 10; + + private RocksDBPersistenceProvider persistenceProvider; + + private SnapshotConfig config; + + private CircularFifoQueue filters; + private CuckooFilter lastAddedFilter; + + // Once this exceeds integer max range, this will give problems. + // Requires max int * FILTER_SIZE * BUCKET_SIZE * ~1.005 transactions (138 billion) + // or max int * 5 min (milestone avg. time) minutes (20428 years) + private Integer highestIndex = -1; + + private int filterSize; + + /** + * Creates a transaction provider with the default filter size {@value #FILTER_SIZE} + */ + public PrunedTransactionProviderImpl() { + this(FILTER_SIZE); + } + + /** + * Creates a transaction provider with a custom filter size + * @param filterSize the size each cuckoo filter will have (before resizing) + */ + public PrunedTransactionProviderImpl(int filterSize) { + this.filterSize = filterSize; + } + + /** + * Starts the PrunedTransactionProvider by reading the current pruned transactions from a persistence provider + * + * @param config The snapshot configuration used for file location + * @return the current instance + * @throws SpentAddressesException if we failed to create a file at the designated location + */ + public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTransactionException { + this.config = config; + try { + this.persistenceProvider = new RocksDBPersistenceProvider( + config.getPrunedTransactionsDbPath(), + config.getPrunedTransactionsDbLogPath(), + 1000, + new HashMap>(1) + {{put("pruned-transactions", Cuckoo.class);}}, null); + this.persistenceProvider.init(); + + filters = new CircularFifoQueue(MAX_FILTERS); + + readPreviousPrunedTransactions(); + } catch (Exception e) { + throw new PrunedTransactionException("There is a problem with accessing previously pruned transactions", e); + } + + return this; + } + + private void readPreviousPrunedTransactions() throws PrunedTransactionException { + if (config.isTestnet()) { + try { + newFilter(); + } catch (Exception e) { + // Ignorable, testnet starts empty, log for debugging + log.warn(e.getMessage()); + } + return; + } + + try { + TreeMap filters = new TreeMap<>(); + + // Load all data from all filters + List bytes = persistenceProvider.loadAllKeysFromTable(Cuckoo.class); + for (byte[] filterData : bytes) { + Cuckoo bucket = new Cuckoo(); + bucket.read(filterData); + + if (MAX_FILTERS < bucket.filterId.getValue()) { + throw new PrunedTransactionException("Database contains more filters then we can store"); + } + + CuckooFilter filter = new CuckooFilterImpl(filterSize, BUCKET_SIZE, FINGER_PRINT_SIZE, bucket.filterBits); + filters.put(bucket.filterId.getValue(), filter); + } + + //Then add all in order, treemap maintains order from lowest to highest key + for (CuckooFilter filter : filters.values()) { + if (null != filter) { + this.filters.add(filter); + } + } + + Entry entry = filters.lastEntry(); + if (null != entry) { + lastAddedFilter = entry.getValue(); + highestIndex = entry.getKey(); + } else { + newFilter(); + } + + } catch (Exception e) { + throw new PrunedTransactionException(e); + } + } + + private void persistFilter(CuckooFilter filter, Integer index) throws Exception { + IntegerIndex intIndex = new IntegerIndex(index); + Cuckoo bucket = new Cuckoo(); + bucket.filterId = intIndex; + bucket.filterBits = filter.getFilterData(); + persistenceProvider.save(bucket, intIndex); + } + + /** + * Checks if our filters contain this transaction hash, starting with the most recent filter + * + * {@inheritDoc} + */ + @Override + public boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException { + byte[] hashBytes = transactionHash.bytes(); + + // last filter added is most recent, thus most likely to be requested + if (lastAddedFilter.contains(hashBytes)) { + return true; + } + + // Loop over other filters in order of recently added, skip first + CuckooFilter[] filterArray = filters.toArray(new CuckooFilter[filters.size()]); + for (int i = filterArray.length - 2; i >=0; i--) { + if (filterArray[i].contains(hashBytes)) { + return true; + } + } + return false; + } + + /** + * Adds a transaction to the latest filter. When this filter is full, saves and creates a new filter. + * + * {@inheritDoc} + * @throws PrunedTransactionException when saving the old (full) filter or deleting the oldest fails + */ + @Override + public void addTransaction(Hash transactionHash) throws PrunedTransactionException { + try { + if (null == lastAddedFilter) { + newFilter(); + } + + if (!lastAddedFilter.add(transactionHash.bytes()) || shouldSwitchFilter()){ + try { + persistFilter(lastAddedFilter, highestIndex); + } catch (Exception e) { + throw new PrunedTransactionException(e); + } + newFilter().add(transactionHash.bytes()); + } + } catch (Exception e) { + throw new PrunedTransactionException(e); + } + } + + /** + * + * {@inheritDoc} + */ + @Override + public void addTransactionBatch(Collection transactionHashes) throws PrunedTransactionException { + // Should we create a new filter when we get this method call? + // It probably implies a new pruned job completed, and these TX would be checked more often then the older ones. + // However, we have less filters to use/faster filter deletion + for (Hash transactionHash : transactionHashes) { + addTransaction(transactionHash); + } + try { + persistFilter(lastAddedFilter, highestIndex); + } catch (Exception e) { + throw new PrunedTransactionException(e); + } + } + + private CuckooFilter newFilter() throws Exception { + if (filters.isAtFullCapacity()) { + log.debug("Removing " + filters.peek()); + persistenceProvider.delete(Cuckoo.class, new IntegerIndex(getLowestIndex())); + } + + highestIndex++; + + // We keep a reference to the last filter to prevent looking it up every time + filters.offer(lastAddedFilter = new CuckooFilterImpl(filterSize, BUCKET_SIZE, FINGER_PRINT_SIZE)); + return lastAddedFilter; + } + + private boolean shouldSwitchFilter() { + // Cuckoo filter speed/accuracy is best when filters stay partially empty + return lastAddedFilter != null && lastAddedFilter.size() > filterSize; + } + + private int getLowestIndex() { + if (highestIndex < MAX_FILTERS) { + return 0; + } + + return highestIndex - MAX_FILTERS; + } +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java new file mode 100644 index 0000000000..15379e7c6e --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java @@ -0,0 +1,197 @@ +package com.iota.iri.service.transactionpruning.impl; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.transactionpruning.PrunedTransactionException; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; +import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier; + +/** + * Verifies the pruned state of a transaction hash by checking each parent the transaction references + * This is done until one transaction is found not to be pruned, or 10 transactions referencing this hash + * are found to be pruned. + */ +public class PrunedTransactionVerifierImpl implements PrunedTransactionVerifier { + + private static final int PRUNED_CERTAIN = 10; + + private PrunedTransactionProvider provider; + + private TransactionRequester requester; + + /** + * List of Hashes who were possibly pruned, but turned out false after verifying + */ + private List verifiedFalse; + + /** + * List of children we have tested per main tx hash + */ + private Map> parents; + + /** + * Map of requested tx and our certainty it being pruned + */ + private Map prunedHashTest; + + /** + * Creates a pruned transaction verifier + * + * @param provider The provider we use to check for a pruned transaction + * @param requester Used to request transaction parents of a hash for verifying the pruned state + */ + public PrunedTransactionVerifierImpl(PrunedTransactionProvider provider, TransactionRequester requester) { + this.provider = provider; + this.requester = requester; + + verifiedFalse = new LinkedList<>(); + prunedHashTest = new HashMap<>(); + } + + /** + * Should be called before adding the transaction hash to ensure the initial hash is pruned + * + * @return true if it could be pruned, false if it definitely wasn't pruned + * @throws PrunedTransactionException If the provider fails to check the transaction + */ + @Override + public boolean isPossiblyPruned(Hash hash) throws PrunedTransactionException { + if (verifiedFalse.contains(hash)) { + return false; + } + return provider.containsTransaction(hash); + } + + /** + * + * {@inheritDoc} + */ + @Override + public boolean isPruned(Hash hash) throws PrunedTransactionException{ + if (verifiedFalse.contains(hash)) { + return false; + } + + if (!prunedHashTest.containsKey(hash)) { + try { + initializeVerify(hash); + } catch (Exception e) { + throw new PrunedTransactionException("Failed to initialize pruned lookup", e); + } + } + + return prunedHashTest.get(hash) >= PRUNED_CERTAIN; + } + + /** + * + * {@inheritDoc} + */ + @Override + public void submitTransaction(TransactionViewModel receivedTransactionViewModel) throws PrunedTransactionException { + Hash parent = receivedTransactionViewModel.getHash(); + Hash child = getChildForParent(parent); + if (child == null || isPruned(child)) { + // We succeeded in the meantime or we were not waiting for this at all + return; + } + + if (isPossiblyPruned(parent)) { + // Add one to the map + prunedHashTest.merge(child, 1, Integer::sum); + + if (isPruned(child)) { + // We succeeded in the meantime. + parents.remove(child); + return; + } + + List parents = getParentsFor(child); + + // It could that they already got referenced through another tx + try { + if (!parents.contains(receivedTransactionViewModel.getBranchTransactionHash())){ + parents.add(receivedTransactionViewModel.getBranchTransactionHash()); + request(receivedTransactionViewModel.getBranchTransactionHash()); + } + if (!parents.contains(receivedTransactionViewModel.getTrunkTransactionHash())){ + parents.add(receivedTransactionViewModel.getTrunkTransactionHash()); + request(receivedTransactionViewModel.getTrunkTransactionHash()); + } + } catch (Exception e) { + // We need to request but failed to do so + throw new PrunedTransactionException(e); + } + } else { + // False positive + clean(child); + verifiedFalse.add(child); + } + } + + /** + * + * {@inheritDoc} + */ + @Override + public boolean waitingForHash(Hash hash) { + return getChildForParent(hash) != null; + } + + private void clean(Hash child) { + prunedHashTest.remove(child); + parents.remove(child); + } + + private void request(Hash hash) throws Exception { + requester.requestTransaction(hash, false); + } + + private List addParentForChild(Hash parent, Hash child) { + List list = getParentsFor(child); + + list.add(parent); + return list; + } + + private List getParentsFor(Hash child) { + if (parents == null) { + parents = new HashMap<>(); + } + + List list; + if (parents.containsKey(child)) { + list = parents.get(child); + } else { + list = new LinkedList<>(); + parents.put(child, list); + } + return list; + } + + private void initializeVerify(Hash hash) throws Exception { + addParentForChild(hash, hash); + prunedHashTest.put(hash, 1); + request(hash); + } + + private Hash getChildForParent(Hash parent) { + if (parents == null) { + return null; + } + + for (Entry> entry : parents.entrySet()) { + if (entry.getValue().contains(parent)) { + return entry.getKey(); + } + } + + return null; + } +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java index 04795c272b..c98efbd5a4 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java @@ -4,6 +4,7 @@ import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; @@ -49,6 +50,11 @@ public abstract class AbstractTransactionPrunerJob implements TransactionPrunerJ */ private Snapshot snapshot; + /** + * Holds a reference to the provider of pruned transactions, which we will use for speeding up old references checks + */ + private PrunedTransactionProvider prunedTransactionProvider; + /** * {@inheritDoc} */ @@ -138,7 +144,23 @@ public TransactionPrunerJobStatus getStatus() { public void setStatus(TransactionPrunerJobStatus status) { this.status = status; } - + + /** + * {@inheritDoc} + */ + @Override + public PrunedTransactionProvider getPrunedProvider() { + return prunedTransactionProvider; + } + + /** + * {@inheritDoc} + */ + @Override + public void setPrunedProvider(PrunedTransactionProvider prunedTransactionProvider) { + this.prunedTransactionProvider = prunedTransactionProvider; + } + /** * {@inheritDoc} */ diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index a32c342286..b507137e76 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -240,6 +241,10 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { }); getTangle().deleteBatch(elementsToDelete); + getPrunedProvider().addTransactionBatch(elementsToDelete + .stream() + .map(a -> (Hash) a.low) + .collect(Collectors.toList())); } catch(Exception e) { throw new TransactionPruningException("failed to cleanup milestone #" + getCurrentIndex(), e); } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java index 780d078267..c29fec8994 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java @@ -4,7 +4,6 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; import com.iota.iri.model.persistables.Transaction; -import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.storage.Indexable; diff --git a/src/main/java/com/iota/iri/utils/Serializer.java b/src/main/java/com/iota/iri/utils/Serializer.java index da14da5380..967e809468 100644 --- a/src/main/java/com/iota/iri/utils/Serializer.java +++ b/src/main/java/com/iota/iri/utils/Serializer.java @@ -37,14 +37,40 @@ public static long getLong(byte[] bytes, int start) { return res; } + /** + * Reads the default amount of bytes for an int(4) and turns it into an int. + * Starts at the beginning of the array + * + * @param bytes The bytes we use to make an int + * @return The created int, or 0 when bytes is null + */ public static int getInteger(byte[] bytes) { return getInteger(bytes, 0); } + + /** + * Reads the default amount of bytes for an int(4) and turns it into an int. + * + * @param bytes The bytes we use to make an int + * @param start The point in the array from which we start reading + * @return The created int, or 0 when bytes is null + */ public static int getInteger(byte[] bytes, int start) { + return getInteger(bytes, start, Integer.BYTES); + } + + /** + * Reads the bytes for the given length, starting at the starting point given. + * + * @param bytes The bytes we use to make an int + * @param start The point in the array from which we start reading + * @param length Amount of bytes to read + * @return The created int, or 0 when bytes is null + */ + public static int getInteger(byte[] bytes, int start, int length) { if(bytes == null) { return 0; } - int length = Integer.BYTES; int res = 0; for (int i=0; i< length;i++) { res |= (bytes[start + i] & 0xFFL) << ((length-i-1) * 8); diff --git a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java index 9807ff2119..e60c6aad0e 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java +++ b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java @@ -1,5 +1,7 @@ package com.iota.iri.utils.datastructure; +import java.util.BitSet; + /** * The Cuckoo Filter is a probabilistic data structure that supports fast set membership testing. * @@ -83,4 +85,10 @@ public interface CuckooFilter { * @return the amount of stored items */ int size(); + + /** + * This method returns a copy of all the bits that make up this filter + * @return The ilter bits + */ + BitSet getFilterData(); } diff --git a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java index 8efa4e2aae..83bd08840c 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java +++ b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java @@ -11,6 +11,7 @@ * This class implements the basic contract of the {@link CuckooFilter}. */ public class CuckooFilterImpl implements CuckooFilter { + /** * The amount of times we try to kick elements when inserting before we consider the index to be too full. */ @@ -117,6 +118,38 @@ public CuckooFilterImpl(int itemCount, int bucketSize, int fingerPrintSize) thro cuckooFilterTable = new CuckooFilterTable(tableSize, bucketSize, fingerPrintSize); } + + /** + * Advanced constructor that allows for fine tuning of the desired filter. + * + * It first saves a reference to the hash function and then checks the parameters - the finger print size cannot + * be bigger than 128 bits because SHA1 generates 160 bits and we use 128 of that for the fingerprint and the rest + * for the index. + * + * After verifying that the passed in parameters are reasonable, we calculate the required size of the + * {@link CuckooFilterTable} by increasing the table size exponentially until we can fit the desired item count with + * a load factor of <= 0.955. Finally we create the {@link CuckooFilterTable} that will hold our data. + * + * NOTE: The actual size will be slightly bigger since the size has to be a power of 2 and take the optimal load + * factor of 0.955 into account. + * + * @param itemCount the minimum amount of items that should fit into the filter + * @param bucketSize the amount of items that can be stored in each bucket + * @param fingerPrintSize the amount of bits per fingerprint (it has to be bigger than 0 and smaller than 128) + * @param filterData The data this filter is initialized with. Must match with the parameters + * @throws IllegalArgumentException if the finger print size is too small or too big + * @throws InternalError if the SHA1 hashing function can not be found with this java version [should never happen] + */ + public CuckooFilterImpl(int itemCount, int bucketSize, int fingerPrintSize, BitSet filterData) throws IllegalArgumentException, + InternalError { + + this(itemCount, bucketSize, fingerPrintSize); + + if (cuckooFilterTable.data.size() != filterData.size()) { + throw new IllegalArgumentException("Filter data does not match filter parameters"); + } + cuckooFilterTable.data = BitSet.valueOf(filterData.toByteArray()); + } /** * {@inheritDoc} @@ -435,6 +468,11 @@ private BitSet generateFingerPrint(byte[] hash) throws IllegalArgumentException return BitSetUtils.convertByteArrayToBitSet(hash, 4, fingerPrintSize); } + @Override + public BitSet getFilterData() { + return (BitSet) cuckooFilterTable.data.clone(); + } + /** * Internal helper class to represent items that are stored in the filter. * diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java index 1ee6f751f6..336b3c5d4e 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/TransactionValidatorTest.java @@ -55,10 +55,10 @@ public static void tearDown() throws Exception { @Test public void testMinMwm() throws InterruptedException { - txValidator.init(false, 5); + txValidator.init(false, 5, null); assertTrue(txValidator.getMinWeightMagnitude() == 13); txValidator.shutdown(); - txValidator.init(false, MAINNET_MWM); + txValidator.init(false, MAINNET_MWM, null); } @Test diff --git a/src/test/java/com/iota/iri/network/NodeTest.java b/src/test/java/com/iota/iri/network/NodeTest.java index 0582463548..4b8467a94f 100644 --- a/src/test/java/com/iota/iri/network/NodeTest.java +++ b/src/test/java/com/iota/iri/network/NodeTest.java @@ -41,7 +41,7 @@ public void setUp() { // set up class under test nodeConfig = Mockito.mock(NodeConfig.class); - classUnderTest = new Node(null, null, null, null, null, null, nodeConfig); + classUnderTest = new Node(null, null, null, null, null, null, nodeConfig, null); // verify config calls in Node constructor verify(nodeConfig).getRequestHashSize(); diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java new file mode 100644 index 0000000000..803e56ef91 --- /dev/null +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java @@ -0,0 +1,99 @@ +package com.iota.iri.service.transactionpruning; + +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import com.iota.iri.TransactionTestUtils; +import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.model.Hash; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; + +public class PrunedTransactionProviderImplTest { + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + public SnapshotConfig config; + + @Rule + public final TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule + public final TemporaryFolder logFolder = new TemporaryFolder(); + + PrunedTransactionProviderImpl provider; + + @Before + public void setUp() throws PrunedTransactionException { + Mockito.when(config.getPrunedTransactionsDbPath()).thenReturn(dbFolder.getRoot().getAbsolutePath()); + Mockito.when(config.getPrunedTransactionsDbLogPath()).thenReturn(logFolder.getRoot().getAbsolutePath()); + + provider = new PrunedTransactionProviderImpl(10000); + provider.init(config); + } + + @After + public void tearDown() { + dbFolder.delete(); + } + + @Test + public void containsMarginalOkayTest() throws PrunedTransactionException { + int size = 10000; + int contains = 0; + + Hash[] hashes = new Hash[size]; + for (int i=0; i size*0.995); + } +} diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java new file mode 100644 index 0000000000..0d035d5e70 --- /dev/null +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java @@ -0,0 +1,130 @@ +package com.iota.iri.service.transactionpruning; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import static org.mockito.Mockito.when; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.stubbing.Answer; + +import com.iota.iri.TransactionTestUtils; +import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionVerifierImpl; + +public class PrunedTransactionVerifierImplTest { + + private static final Hash A = TransactionTestUtils.getTransactionHash(); + private static final Hash B = TransactionTestUtils.getTransactionHash(); + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + public SnapshotConfig config; + + @Rule + public final TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule + public final TemporaryFolder logFolder = new TemporaryFolder(); + + @Mock + PrunedTransactionProviderImpl provider; + + @Mock + TransactionRequester requester; + + PrunedTransactionVerifierImpl verifier; + + @Before + public void setUp() throws PrunedTransactionException { + when(config.getPrunedTransactionsDbPath()).thenReturn(dbFolder.getRoot().getAbsolutePath()); + when(config.getPrunedTransactionsDbLogPath()).thenReturn(logFolder.getRoot().getAbsolutePath()); + + verifier = new PrunedTransactionVerifierImpl(provider, requester); + } + + @After + public void tearDown() { + dbFolder.delete(); + } + + @Test + public void isPossiblyPrunedTest() throws PrunedTransactionException { + when(provider.containsTransaction(A)).thenReturn(true); + + assertTrue("A should be seen as pruned", verifier.isPossiblyPruned(A)); + assertFalse("B should not be seen as pruned", verifier.isPossiblyPruned(B)); + } + + @Test + public void waitingForHashTest() throws PrunedTransactionException { + assertFalse("A should not be seen as definitely pruned", verifier.isPruned(A)); + assertTrue("Verifier should be waiting for TVM for A", verifier.waitingForHash(A)); + + assertFalse("Verifier should not be waitingfor TVM for B", verifier.waitingForHash(B)); + } + + @Test + public void completePruneCheckTest() throws Exception { + // 10 hashes in the pruner is enough to verify + // 0 -> 1 -> 3 -> 7 + // -> 4 -> 8 + // 2 -> 5 -> 9 + // -> 6 + + TransactionViewModel[] tvms = new TransactionViewModel[10]; + tvms[9] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getTransactionTrits()); + tvms[8] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getTransactionTrits()); + tvms[7] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getTransactionTrits()); + tvms[6] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getTransactionTrits()); + + tvms[5] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[9].getHash(), tvms[9].getHash())); + tvms[4] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[8].getHash(), tvms[8].getHash())); + tvms[3] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[7].getHash(), tvms[7].getHash())); + + tvms[2] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[5].getHash(), tvms[6].getHash())); + tvms[1] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[3].getHash(), tvms[4].getHash())); + + tvms[0] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[1].getHash(), tvms[2].getHash())); + + // Mock network request/response on the transactions + for (TransactionViewModel tvm : tvms) { + Mockito.doAnswer(new Answer() { + public Void answer(InvocationOnMock invocation) { + try { + verifier.submitTransaction(tvm); + } catch (PrunedTransactionException e) { + e.printStackTrace(); + } + return null; + } + }).when(requester).requestTransaction(tvm.getHash(), false); + + // Mock as pruned + when(provider.containsTransaction(tvm.getHash())).thenReturn(true); + } + + // Requests are handled directly (DFS), so it should mark pruned! + assertTrue("After checking 10 transactions, hash should be marked as pruned", + verifier.isPruned(tvms[0].getHash())); + } + + private TransactionViewModel c(byte[] trits) { + return TransactionTestUtils.createTransactionFromTrits(trits); + } +}