From d688d9c6ddceb3a9a75d67d3ffff239e4574a59c Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Mon, 1 Apr 2019 02:08:06 +0200 Subject: [PATCH 01/20] Added cuckoofilter implementation details for looading from database --- .../com/iota/iri/conf/BaseIotaConfig.java | 36 ++++++- .../com/iota/iri/conf/SnapshotConfig.java | 13 +++ .../iri/model/persistables/CuckooBucket.java | 47 ++++++++++ .../impl/SpentAddressesProviderImpl.java | 3 +- .../PrunedTransactionException.java | 38 ++++++++ .../PrunedTransactionProvider.java | 36 +++++++ .../impl/PrunedTransactionProviderImpl.java | 93 +++++++++++++++++++ .../jobs/MilestonePrunerJob.java | 4 + .../jobs/UnconfirmedSubtanglePrunerJob.java | 1 - .../iri/utils/datastructure/CuckooFilter.java | 11 +++ .../datastructure/impl/CuckooFilterImpl.java | 20 ++++ 11 files changed, 294 insertions(+), 8 deletions(-) create mode 100644 src/main/java/com/iota/iri/model/persistables/CuckooBucket.java create mode 100644 src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java create mode 100644 src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java create mode 100644 src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index 3756f622a8..cd7efdf078 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -5,7 +5,6 @@ import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.iota.iri.IRI; import com.iota.iri.crypto.SpongeFactory; import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; @@ -109,7 +108,9 @@ public abstract class BaseIotaConfig implements IotaConfig { protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH; protected String spentAddressesDbPath = Defaults.SPENT_ADDRESSES_DB_PATH; protected String spentAddressesDbLogPath = Defaults.SPENT_ADDRESSES_DB_LOG_PATH; - + protected String prunedTransactionsDbPath = Defaults.PRUNED_TRANSACTIONS_DB_PATH; + protected String prunedTransactionsDbLogPath = Defaults.PRUNED_TRANSACTIONS_DB_LOG_PATH; + public BaseIotaConfig() { //empty constructor } @@ -650,13 +651,35 @@ protected void setSpentAddressesDbPath(String spentAddressesDbPath) { public String getSpentAddressesDbLogPath() { return spentAddressesDbLogPath; } - + @JsonProperty @Parameter(names = {"--spent-addresses-db-log-path"}, description = SnapshotConfig.Descriptions.SPENT_ADDRESSES_DB_LOG_PATH) - protected void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) { + protected void setPrunedTransactionDbLogPath(String spentAddressesDbLogPath) { this.spentAddressesDbLogPath = spentAddressesDbLogPath; } - + + @Override + public String getPrunedTransactionsDbLogPath() { + return prunedTransactionsDbLogPath; + } + + @JsonProperty + @Parameter(names = {"--pruned-transactions-db-log-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_LOG_PATH) + protected void setSpentAddressesDbLogPath(String prunedTransactionsDbLogPath) { + this.prunedTransactionsDbLogPath = prunedTransactionsDbLogPath; + } + + @Override + public String getPrunedTransactionsDbPath() { + return prunedTransactionsDbPath; + } + + @JsonProperty + @Parameter(names = {"--pruned-transactions-db-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_PATH) + protected void setPrunedTransactionsDbPath(String prunedTransactionsDbPath) { + this.prunedTransactionsDbPath = prunedTransactionsDbPath; + } + /** * Checks if ZMQ is enabled. * @return true if zmqEnableTcp or zmqEnableIpc is set. @@ -933,6 +956,9 @@ public interface Defaults { int LOCAL_SNAPSHOTS_DEPTH_MIN = 100; String SPENT_ADDRESSES_DB_PATH = "spent-addresses-db"; String SPENT_ADDRESSES_DB_LOG_PATH = "spent-addresses-log"; + + String PRUNED_TRANSACTIONS_DB_LOG_PATH = "spent-addresses-db"; + String PRUNED_TRANSACTIONS_DB_PATH = "spent-addresses-log"; String LOCAL_SNAPSHOTS_BASE_PATH = "mainnet"; String SNAPSHOT_FILE = "/snapshotMainnet.txt"; diff --git a/src/main/java/com/iota/iri/conf/SnapshotConfig.java b/src/main/java/com/iota/iri/conf/SnapshotConfig.java index 33c30fb705..b213c4f8b6 100644 --- a/src/main/java/com/iota/iri/conf/SnapshotConfig.java +++ b/src/main/java/com/iota/iri/conf/SnapshotConfig.java @@ -76,6 +76,16 @@ public interface SnapshotConfig extends Config { */ String getSpentAddressesDbLogPath(); + /** + * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_PATH} + */ + String getPrunedTransactionsDbPath(); + + /** + * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_LOG_PATH} + */ + String getPrunedTransactionsDbLogPath(); + interface Descriptions { String LOCAL_SNAPSHOTS_ENABLED = "Flag that determines if local snapshots are enabled."; @@ -94,5 +104,8 @@ interface Descriptions { "from previous epochs"; String SPENT_ADDRESSES_DB_PATH = "The folder where the spent addresses DB saves its data."; String SPENT_ADDRESSES_DB_LOG_PATH = "The folder where the spent addresses DB saves its logs."; + String PRUNED_TRANSACTIONS_DB_PATH = "The folder where the pruned transactions DB saves its data."; + String PRUNED_TRANSACTIONS_DB_LOG_PATH = "The folder where the pruned transactions DB saves its logs."; } + } diff --git a/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java b/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java new file mode 100644 index 0000000000..3b5b847620 --- /dev/null +++ b/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java @@ -0,0 +1,47 @@ +package com.iota.iri.model.persistables; + +import java.util.BitSet; + +import org.apache.commons.lang3.ArrayUtils; + +import com.iota.iri.model.IntegerIndex; +import com.iota.iri.storage.Persistable; +import com.iota.iri.utils.Serializer; + +public class CuckooBucket implements Persistable { + + public BitSet bucketBits; + public IntegerIndex bucketIndex; + + @Override + public byte[] bytes() { + return ArrayUtils.addAll(bucketIndex.bytes(), bucketBits.toByteArray()); + } + + @Override + public void read(byte[] bytes) { + if(bytes != null) { + bucketIndex = new IntegerIndex(Serializer.getInteger(bytes)); + + bucketBits = new BitSet(bytes.length - 2); + for (int i = 1; i < bytes.length; i++) { + bucketBits.set(i-1, bytes[i]); + } + } + } + + @Override + public byte[] metadata() { + return new byte[0]; + } + + @Override + public void readMetadata(byte[] bytes) { + } + + @Override + public boolean merge() { + return false; + } + +} diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index c59c61dc6e..8bcc7503c7 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -54,8 +54,7 @@ public SpentAddressesProviderImpl init(SnapshotConfig config) {{put("spent-addresses", SpentAddress.class);}}, null); this.rocksDBPersistenceProvider.init(); readPreviousEpochsSpentAddresses(); - } - catch (Exception e) { + } catch (Exception e) { throw new SpentAddressesException("There is a problem with accessing stored spent addresses", e); } return this; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java new file mode 100644 index 0000000000..4e1ea756da --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java @@ -0,0 +1,38 @@ +package com.iota.iri.service.transactionpruning; + +/** + * This class is used to wrap exceptions that are specific to the pruned transaction persistance. + * + * It allows us to distinct between the different kinds of errors that can happen during the execution of the code. + */ +public class PrunedTransactionException extends Exception { + /** + * Constructor of the exception which allows us to provide a specific error message and the cause of the error. + * + * @param message reason why this error occurred + * @param cause wrapped exception that caused this error + */ + public PrunedTransactionException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor of the exception which allows us to provide a specific error message without having an underlying + * cause. + * + * @param message reason why this error occurred + */ + public PrunedTransactionException(String message) { + super(message); + } + + /** + * Constructor of the exception which allows us to wrap the underlying cause of the error without providing a + * specific reason. + * + * @param cause wrapped exception that caused this error + */ + public PrunedTransactionException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java new file mode 100644 index 0000000000..264ba34c9d --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java @@ -0,0 +1,36 @@ +package com.iota.iri.service.transactionpruning; + +import java.util.Collection; + +import com.iota.iri.model.Hash; + +/** + * Find, mark and store pruned transactions + */ +public interface PrunedTransactionProvider { + + /** + * Checks if this transactions has been pruned + * + * @param transactionHash The transaction to check for + * @return true if it is, else false + * @throws PrunedTransactionException If the provider fails to check the transaction + */ + boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException; + + /** + * Mark a transaction as spent. + * + * @param transactionHash the transaction which we want to mark. + * @throws PrunedTransactionException If the provider fails to add the transaction + */ + void addTransaction(Hash transactionHash) throws PrunedTransactionException; + + /** + * Mark all transactions as pruned. + * + * @param transactionHashes The transactions we want to mark + * @throws PrunedTransactionException If the provider fails to add a transaction + */ + void AddTransactionBatch(Collection transactionHashes) throws PrunedTransactionException; +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java new file mode 100644 index 0000000000..8c206b5bd1 --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -0,0 +1,93 @@ +package com.iota.iri.service.transactionpruning.impl; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.model.Hash; +import com.iota.iri.model.HashFactory; +import com.iota.iri.model.persistables.CuckooBucket; +import com.iota.iri.model.persistables.SpentAddress; +import com.iota.iri.service.spentaddresses.SpentAddressesException; +import com.iota.iri.service.spentaddresses.impl.SpentAddressesProviderImpl; +import com.iota.iri.service.transactionpruning.PrunedTransactionException; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; +import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; +import com.iota.iri.utils.datastructure.CuckooFilter; + +public class PrunedTransactionProviderImpl implements PrunedTransactionProvider { + + private static final Logger log = LoggerFactory.getLogger(PrunedTransactionProviderImpl.class); + + private RocksDBPersistenceProvider persistenceProvider; + + private CuckooFilter filter; + + private SnapshotConfig config; + + /** + * Starts the PrunedTransactionProvider by reading the current pruned transactions from a persistence provider + * + * @param config The snapshot configuration used for file location + * @return the current instance + * @throws SpentAddressesException if we failed to create a file at the designated location + */ + public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTransactionException { + this.config = config; + try { + this.persistenceProvider = new RocksDBPersistenceProvider( + config.getPrunedTransactionsDbPath(), + config.getPrunedTransactionsDbLogPath(), + 1000, + new HashMap>(1) + {{put("pruned-transactions", CuckooBucket.class);}}, null); + this.persistenceProvider.init(); + readPreviousPrunedTransactions(); + } + catch (Exception e) { + throw new PrunedTransactionException("There is a problem with accessing stored spent addresses", e); + } + + return this; + } + + private void readPreviousPrunedTransactions() throws PrunedTransactionException { + if (config.isTestnet()) { + return; + } + + try { + for (byte[] bucketData : persistenceProvider.loadAllKeysFromTable(CuckooBucket.class)) { + CuckooBucket bucket = new CuckooBucket(); + bucket.read(bucketData); + filter.update(bucket); + } + } catch (IllegalArgumentException e) { + throw new PrunedTransactionException(e); + } + } + + @Override + public boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException { + return false; + } + + @Override + public void addTransaction(Hash transactionHash) throws PrunedTransactionException { + + } + + @Override + public void AddTransactionBatch(Collection transactionHashes) throws PrunedTransactionException { + + } + +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index b182cc37c9..9838dd0840 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -233,6 +233,10 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { } catch (TransactionPruningException e) { throw new RuntimeException(e); } + } else { + // Confirmed TX, add to data structure + // either add to list, or add a new job to add to the cuckoofilter + } } else if(Milestone.class.equals(element.hi)) { MilestoneViewModel.clear(((IntegerIndex) element.low).getValue()); diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java index 780d078267..c29fec8994 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java @@ -4,7 +4,6 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; import com.iota.iri.model.persistables.Transaction; -import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.storage.Indexable; diff --git a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java index 9807ff2119..6391d521a0 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java +++ b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java @@ -1,5 +1,7 @@ package com.iota.iri.utils.datastructure; +import com.iota.iri.model.persistables.CuckooBucket; + /** * The Cuckoo Filter is a probabilistic data structure that supports fast set membership testing. * @@ -83,4 +85,13 @@ public interface CuckooFilter { * @return the amount of stored items */ int size(); + + /** + * Update a part of this filter with the bucket information supplied. + * Discards any previous data inside this bucket. + * + * @param bucket the bucket data we use to update the filter + * @throws IllegalArgumentException when the bucket does not contain data which is valid for this filter + */ + void update(CuckooBucket bucket) throws IllegalArgumentException ; } diff --git a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java index 8efa4e2aae..46a8d8e292 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java +++ b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java @@ -1,5 +1,6 @@ package com.iota.iri.utils.datastructure.impl; +import com.iota.iri.model.persistables.CuckooBucket; import com.iota.iri.utils.BitSetUtils; import com.iota.iri.utils.datastructure.CuckooFilter; @@ -137,6 +138,25 @@ public boolean add(String item) throws IndexOutOfBoundsException { public boolean add(byte[] item) throws IndexOutOfBoundsException { return add(new CuckooFilterItem(hashFunction.digest(item))); } + /** + * {@inheritDoc} + * + */ + @Override + public void update(CuckooBucket bucket) throws IllegalArgumentException { + int amountInBucket = bucketSize; + if (bucket.bucketBits.length() / amountInBucket > 0) { + + } + + for (int i=0; i < amountInBucket; i++) { + cuckooFilterTable.set(bucket.bucketIndex.getValue(), i, bucket.bucketBits.get( + fingerPrintSize * i, + fingerPrintSize * (i + 1) + )); + + } + } /** * {@inheritDoc} From 989aaba819e7cb0ac9d5d9fa5596c6c165a8d62f Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Fri, 12 Apr 2019 12:08:58 +0200 Subject: [PATCH 02/20] Added PrunedProvider, added it to prunerjobs, updated int parsing serializer --- .../iri/model/persistables/CuckooBucket.java | 48 +++++- .../PrunedTransactionProvider.java | 2 +- .../TransactionPrunerJob.java | 14 ++ .../async/AsyncTransactionPruner.java | 7 + .../impl/PrunedTransactionProviderImpl.java | 143 ++++++++++++++++-- .../jobs/AbstractTransactionPrunerJob.java | 24 ++- .../jobs/HashPruningJob.java | 19 +++ .../jobs/MilestonePrunerJob.java | 5 + .../java/com/iota/iri/utils/Serializer.java | 5 +- .../iri/utils/datastructure/CuckooFilter.java | 7 +- .../datastructure/impl/CuckooFilterImpl.java | 24 ++- .../PrunedTransactionProviderImplTest.java | 69 +++++++++ 12 files changed, 337 insertions(+), 30 deletions(-) create mode 100644 src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java create mode 100644 src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java diff --git a/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java b/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java index 3b5b847620..10f0600f68 100644 --- a/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java +++ b/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java @@ -8,37 +8,75 @@ import com.iota.iri.storage.Persistable; import com.iota.iri.utils.Serializer; +/** + * Persistable to manage the data we get as a result of pruning a milestone and its transactions + */ public class CuckooBucket implements Persistable { + /** + * The filter number it belonged to in previous cycles + */ + public IntegerIndex bucketId; + + /** + * + */ public BitSet bucketBits; public IntegerIndex bucketIndex; + /** + * + * {@inheritDoc} + */ @Override public byte[] bytes() { - return ArrayUtils.addAll(bucketIndex.bytes(), bucketBits.toByteArray()); + byte[] index = bucketIndex.bytes(); + byte[] num = bucketId.bytes(); + return ArrayUtils.addAll(ArrayUtils.addAll(num, index), bucketBits.toByteArray()); } + /** + * Reads a CuckooBucket from the provided bytes. + * First 4 bytes are the bucket id, second 4 are the index inside that bucket + * Rest of the bytes is the bucket data + * + * {@inheritDoc} + */ @Override public void read(byte[] bytes) { if(bytes != null) { - bucketIndex = new IntegerIndex(Serializer.getInteger(bytes)); + bucketId = new IntegerIndex(Serializer.getInteger(bytes, 0)); + bucketIndex = new IntegerIndex(Serializer.getInteger(bytes, 4)); - bucketBits = new BitSet(bytes.length - 2); - for (int i = 1; i < bytes.length; i++) { - bucketBits.set(i-1, bytes[i]); + short start = 8; + bucketBits = new BitSet(bytes.length - start); + for (int i = start; i < bytes.length; i++) { + bucketBits.set(i-start, bytes[i]); } } } + /** + * + * {@inheritDoc} + */ @Override public byte[] metadata() { return new byte[0]; } + /** + * + * {@inheritDoc} + */ @Override public void readMetadata(byte[] bytes) { } + /** + * + * {@inheritDoc} + */ @Override public boolean merge() { return false; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java index 264ba34c9d..ea449b2f22 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java @@ -32,5 +32,5 @@ public interface PrunedTransactionProvider { * @param transactionHashes The transactions we want to mark * @throws PrunedTransactionException If the provider fails to add a transaction */ - void AddTransactionBatch(Collection transactionHashes) throws PrunedTransactionException; + void addTransactionBatch(Collection transactionHashes) throws PrunedTransactionException; } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java index 5398fcf4a5..7d0797405b 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java @@ -112,6 +112,20 @@ public interface TransactionPrunerJob { * @param transactionPrunerJobStatus new execution status of the job */ void setStatus(TransactionPrunerJobStatus transactionPrunerJobStatus); + + /** + * Getter for the pruned transaction maintainer. + * + * @return pruned transaction maintainer of the job. + */ + PrunedTransactionProvider getPrunedProvider(); + + /** + * Setter for the pruned transaction maintainer. + * + * @param prunedTransactionProvider pruned transaction maintainer of the job + */ + void setPrunedProvider(PrunedTransactionProvider prunedTransactionProvider); /** * This method processes the cleanup job and performs the actual pruning. diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index 796afa0c57..99a47edc62 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -4,6 +4,7 @@ import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPruningException; @@ -104,6 +105,11 @@ public class AsyncTransactionPruner implements TransactionPruner { */ private final Map, JobQueue> jobQueues = new HashMap<>(); + /** + * + */ + private PrunedTransactionProvider prunedTransactionProvider; + /** * This method initializes the instance and registers its dependencies.
*
@@ -149,6 +155,7 @@ public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvi @Override public void addJob(TransactionPrunerJob job) throws TransactionPruningException { job.setTransactionPruner(this); + job.setPrunedProvider(prunedTransactionProvider); job.setSpentAddressesService(spentAddressesService); job.setTangle(tangle); job.setTipsViewModel(tipsViewModel); diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java index 8c206b5bd1..10316f9b40 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -1,37 +1,66 @@ package com.iota.iri.service.transactionpruning.impl; -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import org.apache.commons.collections4.queue.CircularFifoQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.iota.iri.conf.SnapshotConfig; import com.iota.iri.model.Hash; -import com.iota.iri.model.HashFactory; import com.iota.iri.model.persistables.CuckooBucket; -import com.iota.iri.model.persistables.SpentAddress; import com.iota.iri.service.spentaddresses.SpentAddressesException; -import com.iota.iri.service.spentaddresses.impl.SpentAddressesProviderImpl; import com.iota.iri.service.transactionpruning.PrunedTransactionException; import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.datastructure.CuckooFilter; +import com.iota.iri.utils.datastructure.impl.CuckooFilterImpl; +/** + * + */ public class PrunedTransactionProviderImpl implements PrunedTransactionProvider { private static final Logger log = LoggerFactory.getLogger(PrunedTransactionProviderImpl.class); - + + private static final int FILTER_SIZE = 200000; + + private static final int MAX_FILTERS = 10; + private RocksDBPersistenceProvider persistenceProvider; + + private SnapshotConfig config; - private CuckooFilter filter; + private CircularFifoQueue filters; + private CuckooFilter lastAddedFilter; - private SnapshotConfig config; + // Once this exceeds integer max range, this will give problems. + private Integer highestIndex = -1; + + private int filterSize; + + /** + * + */ + public PrunedTransactionProviderImpl() { + this(FILTER_SIZE); + } + + /** + * + * @param filterSize + */ + public PrunedTransactionProviderImpl(int filterSize) { + this.filterSize = filterSize; + } /** * Starts the PrunedTransactionProvider by reading the current pruned transactions from a persistence provider @@ -50,10 +79,14 @@ public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTr new HashMap>(1) {{put("pruned-transactions", CuckooBucket.class);}}, null); this.persistenceProvider.init(); + + // 10 * 1.000.000 * 4 hashes we can hold, with a total of 80mb size when max filled. + // The database has overhead for the bucket index, and allows a maximum of 2^7 - 1 filters + filters = new CircularFifoQueue(MAX_FILTERS); + readPreviousPrunedTransactions(); - } - catch (Exception e) { - throw new PrunedTransactionException("There is a problem with accessing stored spent addresses", e); + } catch (Exception e) { + throw new PrunedTransactionException("There is a problem with accessing previously pruned transactions", e); } return this; @@ -61,33 +94,111 @@ public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTr private void readPreviousPrunedTransactions() throws PrunedTransactionException { if (config.isTestnet()) { + newFilter(); return; } try { - for (byte[] bucketData : persistenceProvider.loadAllKeysFromTable(CuckooBucket.class)) { + + TreeMap filters = new TreeMap<>(); + + // Load all data from all filters + List bytes = persistenceProvider.loadAllKeysFromTable(CuckooBucket.class); + for (byte[] bucketData : bytes) { CuckooBucket bucket = new CuckooBucket(); bucket.read(bucketData); - filter.update(bucket); + + if (MAX_FILTERS < bucket.bucketId.getValue()) { + throw new PrunedTransactionException("Database contains more filters then we can store"); + } + + // Find its bucket, or create it if wasnt there + CuckooFilter filter = filters.get(bucket.bucketId.getValue()); + if (null == filter) { + filter = new CuckooFilterImpl(filterSize, 4, 16); + filters.put(bucket.bucketId.getValue(), filter); + } + + // Don't update using the entire CuckooBucket so that packages are separate-able + filter.update(bucket.bucketIndex.getValue(), bucket.bucketBits); + } + + //Then add all in order, treemap maintains order from lowest to highest key + for (CuckooFilter filter : filters.values()) { + if (null != filter) { + this.filters.add(filter); + } + } + + Entry entry = filters.lastEntry(); + if (null != entry) { + lastAddedFilter = entry.getValue(); + highestIndex = entry.getKey(); + } else { + newFilter(); } + } catch (IllegalArgumentException e) { throw new PrunedTransactionException(e); } } + + private void persistFilter(CuckooFilter filter, Integer index) { + + } @Override public boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException { + byte[] hashBytes = transactionHash.bytes(); + + // last filter added is most recent, thus most likely to be requested + CuckooFilter[] filterArray = filters.toArray(new CuckooFilter[filters.size()]); + for (int i = filterArray.length - 1; i >=0; i--) { + if (filterArray[i].contains(hashBytes)) { + return true; + } + } return false; } + /** + * + * {@inheritDoc} + */ @Override public void addTransaction(Hash transactionHash) throws PrunedTransactionException { + if (null == lastAddedFilter) { + newFilter(); + } + if (!lastAddedFilter.add(transactionHash.bytes())){ + newFilter().add(transactionHash.bytes()); + } } + /** + * + * {@inheritDoc} + */ @Override - public void AddTransactionBatch(Collection transactionHashes) throws PrunedTransactionException { - + public void addTransactionBatch(Collection transactionHashes) throws PrunedTransactionException { + // Should we create a new filter when we get this method call? + // It probably implies a new pruned job completed, and these TX would be checked more often then the older ones. + for (Hash transactionHash : transactionHashes) { + addTransaction(transactionHash); + } + persistFilter(lastAddedFilter, highestIndex); } + private CuckooFilter newFilter() { + if (filters.isAtFullCapacity()) { + log.debug("Removing " + filters.peek()); + } + + highestIndex++; + + // We keep a reference to the last filter to prevent looking it up every time + filters.offer(lastAddedFilter = new CuckooFilterImpl(filterSize, 4, 16)); + return lastAddedFilter; + } } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java index ebaf0744df..bf87215f9d 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java @@ -3,6 +3,7 @@ import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; @@ -43,6 +44,11 @@ public abstract class AbstractTransactionPrunerJob implements TransactionPrunerJ */ private Snapshot snapshot; + /** + * Holds a reference to the provider of pruned transactions, which we will use for speeding up old references checks + */ + private PrunedTransactionProvider prunedTransactionProvider; + /** * {@inheritDoc} */ @@ -127,7 +133,23 @@ public TransactionPrunerJobStatus getStatus() { public void setStatus(TransactionPrunerJobStatus status) { this.status = status; } - + + /** + * {@inheritDoc} + */ + @Override + public PrunedTransactionProvider getPrunedProvider() { + return prunedTransactionProvider; + } + + /** + * {@inheritDoc} + */ + @Override + public void setPrunedProvider(PrunedTransactionProvider prunedTransactionProvider) { + this.prunedTransactionProvider = prunedTransactionProvider; + } + /** * {@inheritDoc} */ diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java new file mode 100644 index 0000000000..7a0d6a4fc8 --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java @@ -0,0 +1,19 @@ +package com.iota.iri.service.transactionpruning.jobs; + +import com.iota.iri.service.transactionpruning.TransactionPruningException; + +public class HashPruningJob extends AbstractTransactionPrunerJob { + + public HashPruningJob() { + + } + + @Override + public void process() throws TransactionPruningException { + } + + @Override + public String serialize() { + return null; + } +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index 9838dd0840..62d25fe816 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -244,6 +245,10 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { }); getTangle().deleteBatch(elementsToDelete); + getPrunedProvider().addTransactionBatch(elementsToDelete + .stream() + .map(a -> (Hash) a.low) + .collect(Collectors.toList())); } catch(Exception e) { throw new TransactionPruningException("failed to cleanup milestone #" + getCurrentIndex(), e); } diff --git a/src/main/java/com/iota/iri/utils/Serializer.java b/src/main/java/com/iota/iri/utils/Serializer.java index da14da5380..f46e43edb5 100644 --- a/src/main/java/com/iota/iri/utils/Serializer.java +++ b/src/main/java/com/iota/iri/utils/Serializer.java @@ -41,10 +41,13 @@ public static int getInteger(byte[] bytes) { return getInteger(bytes, 0); } public static int getInteger(byte[] bytes, int start) { + return getInteger(bytes, start, Integer.BYTES); + } + + public static int getInteger(byte[] bytes, int start, int length) { if(bytes == null) { return 0; } - int length = Integer.BYTES; int res = 0; for (int i=0; i< length;i++) { res |= (bytes[start + i] & 0xFFL) << ((length-i-1) * 8); diff --git a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java index 6391d521a0..8cabd828ef 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java +++ b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java @@ -1,5 +1,7 @@ package com.iota.iri.utils.datastructure; +import java.util.BitSet; + import com.iota.iri.model.persistables.CuckooBucket; /** @@ -90,8 +92,9 @@ public interface CuckooFilter { * Update a part of this filter with the bucket information supplied. * Discards any previous data inside this bucket. * - * @param bucket the bucket data we use to update the filter + * @param index The index of the bucket we are updating + * @param bits the bucket data we use to update the filter * @throws IllegalArgumentException when the bucket does not contain data which is valid for this filter */ - void update(CuckooBucket bucket) throws IllegalArgumentException ; + void update(int index, BitSet bits) throws IllegalArgumentException ; } diff --git a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java index 46a8d8e292..6d9f75b601 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java +++ b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java @@ -12,6 +12,11 @@ * This class implements the basic contract of the {@link CuckooFilter}. */ public class CuckooFilterImpl implements CuckooFilter { + + private static int CUR_INDEX = 0; + + int index = CUR_INDEX ++; + /** * The amount of times we try to kick elements when inserting before we consider the index to be too full. */ @@ -143,14 +148,20 @@ public boolean add(byte[] item) throws IndexOutOfBoundsException { * */ @Override - public void update(CuckooBucket bucket) throws IllegalArgumentException { + public void update(int index, BitSet bits) throws IllegalArgumentException { int amountInBucket = bucketSize; - if (bucket.bucketBits.length() / amountInBucket > 0) { - + if (bits.length() % fingerPrintSize != 0) { + throw new IllegalArgumentException("Provided bits do not match fingerprint scheme"); + } else if (bits.length() % fingerPrintSize > amountInBucket) { + throw new IllegalArgumentException("Provided fingerprint data will overflow the bucket"); + } else if (index > tableSize * 0.955) { + throw new IllegalArgumentException("Provided bucket exceeds filter size"); + } else if (false) { + // Can we recover input in any case when expected input does not match given? } for (int i=0; i < amountInBucket; i++) { - cuckooFilterTable.set(bucket.bucketIndex.getValue(), i, bucket.bucketBits.get( + cuckooFilterTable.set(index, i, bits.get( fingerPrintSize * i, fingerPrintSize * (i + 1) )); @@ -606,4 +617,9 @@ public CuckooFilterTable delete(int bucketIndex, int slotIndex) { return set(bucketIndex, slotIndex, null); } } + + @Override + public String toString() { + return index +" "; + } } diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java new file mode 100644 index 0000000000..e609c4654d --- /dev/null +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java @@ -0,0 +1,69 @@ +package com.iota.iri.service.transactionpruning; + +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import com.iota.iri.TransactionTestUtils; +import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.model.Hash; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; + +public class PrunedTransactionProviderImplTest { + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + public SnapshotConfig config; + + @Rule + public final TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule + public final TemporaryFolder logFolder = new TemporaryFolder(); + + PrunedTransactionProviderImpl provider; + + @Before + public void setUp() throws PrunedTransactionException { + Mockito.when(config.getPrunedTransactionsDbPath()).thenReturn(dbFolder.getRoot().getAbsolutePath()); + Mockito.when(config.getPrunedTransactionsDbLogPath()).thenReturn(logFolder.getRoot().getAbsolutePath()); + + // Filter size of 10, for easier testing + provider = new PrunedTransactionProviderImpl(10); + provider.init(config); + } + + @After + public void tearDown() { + dbFolder.delete(); + } + + @Test + public void test() throws PrunedTransactionException { + Hash randomHash = TransactionTestUtils.getRandomTransactionHash(); + provider.addTransaction(randomHash); + for (int i=0; i<100; i++) { + provider.addTransaction(TransactionTestUtils.getRandomTransactionHash()); + } + + int contains = 0; + for (int i=0; i<100; i++) { + if (provider.containsTransaction(randomHash)){ + contains++; + } + } + + System.out.println(contains); + assertTrue("Provider should contain the hash", contains > 90); + } +} From df1a2ddd59dc0bf8a5b995a2862963f482fb5c18 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Sun, 14 Apr 2019 19:58:41 +0200 Subject: [PATCH 03/20] Added saving filters --- .../{CuckooBucket.java => Cuckoo.java} | 21 ++--- .../impl/PrunedTransactionProviderImpl.java | 91 ++++++++++++------- .../iri/utils/datastructure/CuckooFilter.java | 12 +-- .../datastructure/impl/CuckooFilterImpl.java | 73 ++++++++------- 4 files changed, 115 insertions(+), 82 deletions(-) rename src/main/java/com/iota/iri/model/persistables/{CuckooBucket.java => Cuckoo.java} (68%) diff --git a/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java similarity index 68% rename from src/main/java/com/iota/iri/model/persistables/CuckooBucket.java rename to src/main/java/com/iota/iri/model/persistables/Cuckoo.java index 10f0600f68..780f18bfa4 100644 --- a/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java +++ b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java @@ -11,18 +11,17 @@ /** * Persistable to manage the data we get as a result of pruning a milestone and its transactions */ -public class CuckooBucket implements Persistable { +public class Cuckoo implements Persistable { /** * The filter number it belonged to in previous cycles */ - public IntegerIndex bucketId; + public IntegerIndex filterId; /** * */ - public BitSet bucketBits; - public IntegerIndex bucketIndex; + public BitSet filterBits; /** * @@ -30,9 +29,8 @@ public class CuckooBucket implements Persistable { */ @Override public byte[] bytes() { - byte[] index = bucketIndex.bytes(); - byte[] num = bucketId.bytes(); - return ArrayUtils.addAll(ArrayUtils.addAll(num, index), bucketBits.toByteArray()); + byte[] num = filterId.bytes(); + return ArrayUtils.addAll(num, filterBits.toByteArray()); } /** @@ -45,13 +43,12 @@ public byte[] bytes() { @Override public void read(byte[] bytes) { if(bytes != null) { - bucketId = new IntegerIndex(Serializer.getInteger(bytes, 0)); - bucketIndex = new IntegerIndex(Serializer.getInteger(bytes, 4)); + filterId = new IntegerIndex(Serializer.getInteger(bytes, 0)); - short start = 8; - bucketBits = new BitSet(bytes.length - start); + short start = 4; + filterBits = new BitSet(bytes.length - start); for (int i = start; i < bytes.length; i++) { - bucketBits.set(i-start, bytes[i]); + filterBits.set(i-start, bytes[i]); } } } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java index 10316f9b40..aa3087d357 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -15,10 +15,12 @@ import com.iota.iri.conf.SnapshotConfig; import com.iota.iri.model.Hash; -import com.iota.iri.model.persistables.CuckooBucket; +import com.iota.iri.model.IntegerIndex; +import com.iota.iri.model.persistables.Cuckoo; import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.transactionpruning.PrunedTransactionException; import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; +import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.datastructure.CuckooFilter; @@ -77,11 +79,9 @@ public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTr config.getPrunedTransactionsDbLogPath(), 1000, new HashMap>(1) - {{put("pruned-transactions", CuckooBucket.class);}}, null); + {{put("pruned-transactions", Cuckoo.class);}}, null); this.persistenceProvider.init(); - // 10 * 1.000.000 * 4 hashes we can hold, with a total of 80mb size when max filled. - // The database has overhead for the bucket index, and allows a maximum of 2^7 - 1 filters filters = new CircularFifoQueue(MAX_FILTERS); readPreviousPrunedTransactions(); @@ -94,33 +94,30 @@ public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTr private void readPreviousPrunedTransactions() throws PrunedTransactionException { if (config.isTestnet()) { - newFilter(); + try { + newFilter(); + } catch (Exception e) { + // Ignorable, testnet starts empty, log for debugging + log.warn(e.getMessage()); + } return; } try { - TreeMap filters = new TreeMap<>(); // Load all data from all filters - List bytes = persistenceProvider.loadAllKeysFromTable(CuckooBucket.class); - for (byte[] bucketData : bytes) { - CuckooBucket bucket = new CuckooBucket(); - bucket.read(bucketData); + List bytes = persistenceProvider.loadAllKeysFromTable(Cuckoo.class); + for (byte[] filterData : bytes) { + Cuckoo bucket = new Cuckoo(); + bucket.read(filterData); - if (MAX_FILTERS < bucket.bucketId.getValue()) { + if (MAX_FILTERS < bucket.filterId.getValue()) { throw new PrunedTransactionException("Database contains more filters then we can store"); } - // Find its bucket, or create it if wasnt there - CuckooFilter filter = filters.get(bucket.bucketId.getValue()); - if (null == filter) { - filter = new CuckooFilterImpl(filterSize, 4, 16); - filters.put(bucket.bucketId.getValue(), filter); - } - - // Don't update using the entire CuckooBucket so that packages are separate-able - filter.update(bucket.bucketIndex.getValue(), bucket.bucketBits); + CuckooFilter filter = new CuckooFilterImpl(filterSize, 4, 16, bucket.filterBits); + filters.put(bucket.filterId.getValue(), filter); } //Then add all in order, treemap maintains order from lowest to highest key @@ -138,15 +135,23 @@ private void readPreviousPrunedTransactions() throws PrunedTransactionException newFilter(); } - } catch (IllegalArgumentException e) { + } catch (Exception e) { throw new PrunedTransactionException(e); } } - private void persistFilter(CuckooFilter filter, Integer index) { - + private void persistFilter(CuckooFilter filter, Integer index) throws Exception { + IntegerIndex intIndex = new IntegerIndex(index); + Cuckoo bucket = new Cuckoo(); + bucket.filterId = intIndex; + bucket.filterBits = filter.getFilterData(); + persistenceProvider.save(bucket, intIndex); } + /** + * + * {@inheritDoc} + */ @Override public boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException { byte[] hashBytes = transactionHash.bytes(); @@ -162,17 +167,28 @@ public boolean containsTransaction(Hash transactionHash) throws PrunedTransactio } /** + * Adds a transaction to the latest filter. When this filter is full, saves and creates a new filter. * * {@inheritDoc} + * @throws PrunedTransactionException when saving the old (full) filter or deleting the oldest fails */ @Override public void addTransaction(Hash transactionHash) throws PrunedTransactionException { - if (null == lastAddedFilter) { - newFilter(); - } - - if (!lastAddedFilter.add(transactionHash.bytes())){ - newFilter().add(transactionHash.bytes()); + try { + if (null == lastAddedFilter) { + newFilter(); + } + + if (!lastAddedFilter.add(transactionHash.bytes())){ + try { + persistFilter(lastAddedFilter, highestIndex); + } catch (Exception e) { + throw new PrunedTransactionException(e); + } + newFilter().add(transactionHash.bytes()); + } + } catch (Exception e) { + throw new PrunedTransactionException(e); } } @@ -187,12 +203,17 @@ public void addTransactionBatch(Collection transactionHashes) throws Prune for (Hash transactionHash : transactionHashes) { addTransaction(transactionHash); } - persistFilter(lastAddedFilter, highestIndex); + try { + persistFilter(lastAddedFilter, highestIndex); + } catch (Exception e) { + throw new PrunedTransactionException(e); + } } - private CuckooFilter newFilter() { + private CuckooFilter newFilter() throws Exception { if (filters.isAtFullCapacity()) { log.debug("Removing " + filters.peek()); + persistenceProvider.delete(Cuckoo.class, new IntegerIndex(getLowestIndex())); } highestIndex++; @@ -201,4 +222,12 @@ private CuckooFilter newFilter() { filters.offer(lastAddedFilter = new CuckooFilterImpl(filterSize, 4, 16)); return lastAddedFilter; } + + private int getLowestIndex() { + if (highestIndex < MAX_FILTERS) { + return 0; + } + + return highestIndex - MAX_FILTERS; + } } diff --git a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java index 8cabd828ef..4c69ee31c3 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java +++ b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java @@ -2,7 +2,7 @@ import java.util.BitSet; -import com.iota.iri.model.persistables.CuckooBucket; +import com.iota.iri.model.persistables.Cuckoo; /** * The Cuckoo Filter is a probabilistic data structure that supports fast set membership testing. @@ -89,12 +89,8 @@ public interface CuckooFilter { int size(); /** - * Update a part of this filter with the bucket information supplied. - * Discards any previous data inside this bucket. - * - * @param index The index of the bucket we are updating - * @param bits the bucket data we use to update the filter - * @throws IllegalArgumentException when the bucket does not contain data which is valid for this filter + * This method returns a copy of all the bits that make up this filter + * @return The ilter bits */ - void update(int index, BitSet bits) throws IllegalArgumentException ; + BitSet getFilterData(); } diff --git a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java index 6d9f75b601..e7ff51c68c 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java +++ b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java @@ -1,6 +1,5 @@ package com.iota.iri.utils.datastructure.impl; -import com.iota.iri.model.persistables.CuckooBucket; import com.iota.iri.utils.BitSetUtils; import com.iota.iri.utils.datastructure.CuckooFilter; @@ -123,6 +122,38 @@ public CuckooFilterImpl(int itemCount, int bucketSize, int fingerPrintSize) thro cuckooFilterTable = new CuckooFilterTable(tableSize, bucketSize, fingerPrintSize); } + + /** + * Advanced constructor that allows for fine tuning of the desired filter. + * + * It first saves a reference to the hash function and then checks the parameters - the finger print size cannot + * be bigger than 128 bits because SHA1 generates 160 bits and we use 128 of that for the fingerprint and the rest + * for the index. + * + * After verifying that the passed in parameters are reasonable, we calculate the required size of the + * {@link CuckooFilterTable} by increasing the table size exponentially until we can fit the desired item count with + * a load factor of <= 0.955. Finally we create the {@link CuckooFilterTable} that will hold our data. + * + * NOTE: The actual size will be slightly bigger since the size has to be a power of 2 and take the optimal load + * factor of 0.955 into account. + * + * @param itemCount the minimum amount of items that should fit into the filter + * @param bucketSize the amount of items that can be stored in each bucket + * @param fingerPrintSize the amount of bits per fingerprint (it has to be bigger than 0 and smaller than 128) + * @param filterData The data this filter is initialized with. Must match with the parameters + * @throws IllegalArgumentException if the finger print size is too small or too big + * @throws InternalError if the SHA1 hashing function can not be found with this java version [should never happen] + */ + public CuckooFilterImpl(int itemCount, int bucketSize, int fingerPrintSize, BitSet filterData) throws IllegalArgumentException, + InternalError { + + this(itemCount, bucketSize, fingerPrintSize); + + if (cuckooFilterTable.data.size() != filterData.size()) { + throw new IllegalArgumentException("Filter data does not match filter parameters"); + } + cuckooFilterTable.data = BitSet.valueOf(filterData.toByteArray()); + } /** * {@inheritDoc} @@ -143,31 +174,6 @@ public boolean add(String item) throws IndexOutOfBoundsException { public boolean add(byte[] item) throws IndexOutOfBoundsException { return add(new CuckooFilterItem(hashFunction.digest(item))); } - /** - * {@inheritDoc} - * - */ - @Override - public void update(int index, BitSet bits) throws IllegalArgumentException { - int amountInBucket = bucketSize; - if (bits.length() % fingerPrintSize != 0) { - throw new IllegalArgumentException("Provided bits do not match fingerprint scheme"); - } else if (bits.length() % fingerPrintSize > amountInBucket) { - throw new IllegalArgumentException("Provided fingerprint data will overflow the bucket"); - } else if (index > tableSize * 0.955) { - throw new IllegalArgumentException("Provided bucket exceeds filter size"); - } else if (false) { - // Can we recover input in any case when expected input does not match given? - } - - for (int i=0; i < amountInBucket; i++) { - cuckooFilterTable.set(index, i, bits.get( - fingerPrintSize * i, - fingerPrintSize * (i + 1) - )); - - } - } /** * {@inheritDoc} @@ -465,6 +471,16 @@ private BitSet generateFingerPrint(byte[] hash) throws IllegalArgumentException // do a simple conversion of the byte array to a BitSet of the desired length return BitSetUtils.convertByteArrayToBitSet(hash, 4, fingerPrintSize); } + + @Override + public String toString() { + return index + ""; + } + + @Override + public BitSet getFilterData() { + return (BitSet) cuckooFilterTable.data.clone(); + } /** * Internal helper class to represent items that are stored in the filter. @@ -617,9 +633,4 @@ public CuckooFilterTable delete(int bucketIndex, int slotIndex) { return set(bucketIndex, slotIndex, null); } } - - @Override - public String toString() { - return index +" "; - } } From 8c2ade28cb7ca9457af733e73250917c0380d3a5 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Sun, 14 Apr 2019 20:47:56 +0200 Subject: [PATCH 04/20] Added tests --- .../impl/PrunedTransactionProviderImpl.java | 40 +++++++++----- .../PrunedTransactionProviderImplTest.java | 54 ++++++++++++++----- 2 files changed, 69 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java index aa3087d357..9493dee683 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -2,11 +2,8 @@ import java.util.Collection; import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Map.Entry; -import java.util.Set; import java.util.TreeMap; import org.apache.commons.collections4.queue.CircularFifoQueue; @@ -20,7 +17,6 @@ import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.transactionpruning.PrunedTransactionException; import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; -import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.datastructure.CuckooFilter; @@ -33,6 +29,10 @@ public class PrunedTransactionProviderImpl implements PrunedTransactionProvider private static final Logger log = LoggerFactory.getLogger(PrunedTransactionProviderImpl.class); + private static final int BUCKET_SIZE = 4; + + private static final int FINGER_PRINT_SIZE = 16; + private static final int FILTER_SIZE = 200000; private static final int MAX_FILTERS = 10; @@ -44,21 +44,23 @@ public class PrunedTransactionProviderImpl implements PrunedTransactionProvider private CircularFifoQueue filters; private CuckooFilter lastAddedFilter; - // Once this exceeds integer max range, this will give problems. + // Once this exceeds integer max range, this will give problems. + // Requires max int * FILTER_SIZE * BUCKET_SIZE * ~1.005 transactions (138 billion) + // or max int * 5 min (milestone avg. time) minutes (20428 years) private Integer highestIndex = -1; private int filterSize; /** - * + * Creates a transaction provider with the default filter size {@value #FILTER_SIZE} */ public PrunedTransactionProviderImpl() { this(FILTER_SIZE); } /** - * - * @param filterSize + * Creates a transaction provider with a custom filter size + * @param filterSize the size each cuckoo filter will have (before resizing) */ public PrunedTransactionProviderImpl(int filterSize) { this.filterSize = filterSize; @@ -116,7 +118,7 @@ private void readPreviousPrunedTransactions() throws PrunedTransactionException throw new PrunedTransactionException("Database contains more filters then we can store"); } - CuckooFilter filter = new CuckooFilterImpl(filterSize, 4, 16, bucket.filterBits); + CuckooFilter filter = new CuckooFilterImpl(filterSize, BUCKET_SIZE, FINGER_PRINT_SIZE, bucket.filterBits); filters.put(bucket.filterId.getValue(), filter); } @@ -149,16 +151,22 @@ private void persistFilter(CuckooFilter filter, Integer index) throws Exception } /** + * Checks if our filters contain this transaction hash, starting with the most recent filter * * {@inheritDoc} */ @Override public boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException { byte[] hashBytes = transactionHash.bytes(); - + // last filter added is most recent, thus most likely to be requested + if (lastAddedFilter.contains(hashBytes)) { + return true; + } + + // Loop over other filters in order of recently added, skip first CuckooFilter[] filterArray = filters.toArray(new CuckooFilter[filters.size()]); - for (int i = filterArray.length - 1; i >=0; i--) { + for (int i = filterArray.length - 2; i >=0; i--) { if (filterArray[i].contains(hashBytes)) { return true; } @@ -179,7 +187,7 @@ public void addTransaction(Hash transactionHash) throws PrunedTransactionExcepti newFilter(); } - if (!lastAddedFilter.add(transactionHash.bytes())){ + if (!lastAddedFilter.add(transactionHash.bytes()) || shouldSwitchFilter()){ try { persistFilter(lastAddedFilter, highestIndex); } catch (Exception e) { @@ -200,6 +208,7 @@ public void addTransaction(Hash transactionHash) throws PrunedTransactionExcepti public void addTransactionBatch(Collection transactionHashes) throws PrunedTransactionException { // Should we create a new filter when we get this method call? // It probably implies a new pruned job completed, and these TX would be checked more often then the older ones. + // However, we have less filters to use/faster filter deletion for (Hash transactionHash : transactionHashes) { addTransaction(transactionHash); } @@ -219,10 +228,15 @@ private CuckooFilter newFilter() throws Exception { highestIndex++; // We keep a reference to the last filter to prevent looking it up every time - filters.offer(lastAddedFilter = new CuckooFilterImpl(filterSize, 4, 16)); + filters.offer(lastAddedFilter = new CuckooFilterImpl(filterSize, BUCKET_SIZE, FINGER_PRINT_SIZE)); return lastAddedFilter; } + private boolean shouldSwitchFilter() { + // Cuckoo filter speed/accuracy is best when filters stay partially empty + return lastAddedFilter != null && lastAddedFilter.size() > filterSize; + } + private int getLowestIndex() { if (highestIndex < MAX_FILTERS) { return 0; diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java index e609c4654d..99e9a0470d 100644 --- a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java @@ -38,8 +38,7 @@ public void setUp() throws PrunedTransactionException { Mockito.when(config.getPrunedTransactionsDbPath()).thenReturn(dbFolder.getRoot().getAbsolutePath()); Mockito.when(config.getPrunedTransactionsDbLogPath()).thenReturn(logFolder.getRoot().getAbsolutePath()); - // Filter size of 10, for easier testing - provider = new PrunedTransactionProviderImpl(10); + provider = new PrunedTransactionProviderImpl(10000); provider.init(config); } @@ -49,21 +48,52 @@ public void tearDown() { } @Test - public void test() throws PrunedTransactionException { - Hash randomHash = TransactionTestUtils.getRandomTransactionHash(); - provider.addTransaction(randomHash); - for (int i=0; i<100; i++) { - provider.addTransaction(TransactionTestUtils.getRandomTransactionHash()); + public void containsMarginalOkayTest() throws PrunedTransactionException { + int size = 10000; + int contains = 0; + + Hash[] hashes = new Hash[size]; + for (int i=0; i 90); + // 0.05% margin of spents for unknowns + assertTrue("Provider should only contain the added hash", contains < size*1.0005); + } + + @Test + public void deletedFiltersTest() throws PrunedTransactionException { + int size = 100; + Hash[] hashes = new Hash[size]; + for (int i=0; i size*0.995); } } From 54fd525cf0e57a3a2858b69ff614a3aebc285771 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Mon, 15 Apr 2019 20:33:11 +0200 Subject: [PATCH 05/20] Added extra javadoc --- .../impl/PrunedTransactionProviderImpl.java | 15 ++++++++++++++- .../PrunedTransactionProviderImplTest.java | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java index 9493dee683..a7c1ab7f53 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -23,18 +23,31 @@ import com.iota.iri.utils.datastructure.impl.CuckooFilterImpl; /** - * + * Implementation of a pruned transaction provider which uses a cuckoo filter to store pruned hashes */ public class PrunedTransactionProviderImpl implements PrunedTransactionProvider { private static final Logger log = LoggerFactory.getLogger(PrunedTransactionProviderImpl.class); + /** + * The amount of fingerprints each bucket keeps. + * More than 4 + */ private static final int BUCKET_SIZE = 4; + /** + * The amount of bits a hash gets transformed to (Higher is better, but uses more storage) + */ private static final int FINGER_PRINT_SIZE = 16; + /** + * The estimated amount of fingerprints each filter should hold (Will be scaled by cuckoo impl.) + */ private static final int FILTER_SIZE = 200000; + /** + * The maximum amount of filters we have in use at any given time + */ private static final int MAX_FILTERS = 10; private RocksDBPersistenceProvider persistenceProvider; diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java index 99e9a0470d..6ba62a86f3 100644 --- a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java @@ -86,8 +86,8 @@ public void deletedFiltersTest() throws PrunedTransactionException { for (int i=0; i<10000*10; i++) { provider.addTransaction(TransactionTestUtils.getRandomTransactionHash()); } - + // by now, we have added 10 full filters, so the first 100 should not exist anymore (deleted) int notContains = 0; for (int i=0; i Date: Wed, 24 Apr 2019 15:54:21 +0200 Subject: [PATCH 06/20] Added javadoc and cleaned import --- src/main/java/com/iota/iri/model/persistables/Cuckoo.java | 2 +- .../java/com/iota/iri/utils/datastructure/CuckooFilter.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/com/iota/iri/model/persistables/Cuckoo.java b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java index 780f18bfa4..6e397c759a 100644 --- a/src/main/java/com/iota/iri/model/persistables/Cuckoo.java +++ b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java @@ -19,7 +19,7 @@ public class Cuckoo implements Persistable { public IntegerIndex filterId; /** - * + * The bits that make up the CF */ public BitSet filterBits; diff --git a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java index 4c69ee31c3..e60c6aad0e 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java +++ b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java @@ -2,8 +2,6 @@ import java.util.BitSet; -import com.iota.iri.model.persistables.Cuckoo; - /** * The Cuckoo Filter is a probabilistic data structure that supports fast set membership testing. * From d1b1d2a6cf05420006a355f3e874738941c92d6b Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 24 Apr 2019 15:59:06 +0200 Subject: [PATCH 07/20] Fixed naming error in config --- src/main/java/com/iota/iri/conf/BaseIotaConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index cd7efdf078..7fb07657de 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -654,7 +654,7 @@ public String getSpentAddressesDbLogPath() { @JsonProperty @Parameter(names = {"--spent-addresses-db-log-path"}, description = SnapshotConfig.Descriptions.SPENT_ADDRESSES_DB_LOG_PATH) - protected void setPrunedTransactionDbLogPath(String spentAddressesDbLogPath) { + protected void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) { this.spentAddressesDbLogPath = spentAddressesDbLogPath; } @@ -665,7 +665,7 @@ public String getPrunedTransactionsDbLogPath() { @JsonProperty @Parameter(names = {"--pruned-transactions-db-log-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_LOG_PATH) - protected void setSpentAddressesDbLogPath(String prunedTransactionsDbLogPath) { + protected void setPrunedTransactionDbLogPath(String prunedTransactionsDbLogPath) { this.prunedTransactionsDbLogPath = prunedTransactionsDbLogPath; } From 998d8ec2c3d0c06b0b671c9d8e9a70ffa7a227b2 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 24 Apr 2019 16:04:13 +0200 Subject: [PATCH 08/20] Removed/updated more comments, deleted unused file --- .../async/AsyncTransactionPruner.java | 2 +- .../impl/PrunedTransactionProviderImpl.java | 2 +- .../jobs/HashPruningJob.java | 19 ------------------- .../jobs/MilestonePrunerJob.java | 4 ---- 4 files changed, 2 insertions(+), 25 deletions(-) delete mode 100644 src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index 99a47edc62..6b4a9c3d95 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -106,7 +106,7 @@ public class AsyncTransactionPruner implements TransactionPruner { private final Map, JobQueue> jobQueues = new HashMap<>(); /** - * + * Provider for managing transactions we delete from the database in an optimized data structure */ private PrunedTransactionProvider prunedTransactionProvider; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java index a7c1ab7f53..afc197acd9 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -31,7 +31,7 @@ public class PrunedTransactionProviderImpl implements PrunedTransactionProvider /** * The amount of fingerprints each bucket keeps. - * More than 4 + * More than 4 will be slower in performance, whilst the extra space it gets us is not needed. */ private static final int BUCKET_SIZE = 4; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java deleted file mode 100644 index 7a0d6a4fc8..0000000000 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.iota.iri.service.transactionpruning.jobs; - -import com.iota.iri.service.transactionpruning.TransactionPruningException; - -public class HashPruningJob extends AbstractTransactionPrunerJob { - - public HashPruningJob() { - - } - - @Override - public void process() throws TransactionPruningException { - } - - @Override - public String serialize() { - return null; - } -} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index 62d25fe816..a6eff17628 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -234,10 +234,6 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { } catch (TransactionPruningException e) { throw new RuntimeException(e); } - } else { - // Confirmed TX, add to data structure - // either add to list, or add a new job to add to the cuckoofilter - } } else if(Milestone.class.equals(element.hi)) { MilestoneViewModel.clear(((IntegerIndex) element.low).getValue()); From eafce427ccae08aefb2cfc38f3c5bf14d840a48e Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 24 Apr 2019 20:43:58 +0200 Subject: [PATCH 09/20] Documented serializer --- .../java/com/iota/iri/utils/Serializer.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/main/java/com/iota/iri/utils/Serializer.java b/src/main/java/com/iota/iri/utils/Serializer.java index f46e43edb5..967e809468 100644 --- a/src/main/java/com/iota/iri/utils/Serializer.java +++ b/src/main/java/com/iota/iri/utils/Serializer.java @@ -37,13 +37,36 @@ public static long getLong(byte[] bytes, int start) { return res; } + /** + * Reads the default amount of bytes for an int(4) and turns it into an int. + * Starts at the beginning of the array + * + * @param bytes The bytes we use to make an int + * @return The created int, or 0 when bytes is null + */ public static int getInteger(byte[] bytes) { return getInteger(bytes, 0); } + + /** + * Reads the default amount of bytes for an int(4) and turns it into an int. + * + * @param bytes The bytes we use to make an int + * @param start The point in the array from which we start reading + * @return The created int, or 0 when bytes is null + */ public static int getInteger(byte[] bytes, int start) { return getInteger(bytes, start, Integer.BYTES); } + /** + * Reads the bytes for the given length, starting at the starting point given. + * + * @param bytes The bytes we use to make an int + * @param start The point in the array from which we start reading + * @param length Amount of bytes to read + * @return The created int, or 0 when bytes is null + */ public static int getInteger(byte[] bytes, int start, int length) { if(bytes == null) { return 0; From 9f64545dd0b23bd5758631f0275f86656cda9797 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 24 Apr 2019 20:44:13 +0200 Subject: [PATCH 10/20] Removed unused index in CF --- .../iri/utils/datastructure/impl/CuckooFilterImpl.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java index e7ff51c68c..83bd08840c 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java +++ b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java @@ -12,10 +12,6 @@ */ public class CuckooFilterImpl implements CuckooFilter { - private static int CUR_INDEX = 0; - - int index = CUR_INDEX ++; - /** * The amount of times we try to kick elements when inserting before we consider the index to be too full. */ @@ -471,11 +467,6 @@ private BitSet generateFingerPrint(byte[] hash) throws IllegalArgumentException // do a simple conversion of the byte array to a BitSet of the desired length return BitSetUtils.convertByteArrayToBitSet(hash, 4, fingerPrintSize); } - - @Override - public String toString() { - return index + ""; - } @Override public BitSet getFilterData() { From a35f8410adb0a38d939484ddac703146c4c9cde4 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Tue, 14 May 2019 18:19:15 +0200 Subject: [PATCH 11/20] Fixed test method usage --- .../PrunedTransactionProviderImplTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java index 6ba62a86f3..803e56ef91 100644 --- a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java @@ -54,7 +54,7 @@ public void containsMarginalOkayTest() throws PrunedTransactionException { Hash[] hashes = new Hash[size]; for (int i=0; i Date: Tue, 16 Apr 2019 00:59:16 +0200 Subject: [PATCH 12/20] Added Transactionpruner to the required services --- src/main/java/com/iota/iri/Iota.java | 24 ++++++++++++++++--- .../async/AsyncTransactionPruner.java | 6 ++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 48ed6875fe..2c60c67306 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -21,8 +21,11 @@ import com.iota.iri.service.spentaddresses.impl.SpentAddressesServiceImpl; import com.iota.iri.service.tipselection.*; import com.iota.iri.service.tipselection.impl.*; +import com.iota.iri.service.transactionpruning.PrunedTransactionException; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; import com.iota.iri.storage.*; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Pair; @@ -95,6 +98,8 @@ public class Iota { public final TransactionRequesterWorkerImpl transactionRequesterWorker; + public final PrunedTransactionProviderImpl prunedTransactionProvider; + public final BundleValidator bundleValidator; public final Tangle tangle; @@ -108,6 +113,7 @@ public class Iota { public final TipsViewModel tipsViewModel; public final TipSelector tipsSelector; + /** * Initializes the latest snapshot and then creates all services needed to run an IOTA node. * @@ -115,8 +121,9 @@ public class Iota { * @throws TransactionPruningException If the TransactionPruner could not restore its state. * @throws SnapshotException If the Snapshot fails to initialize. * This can happen if the snapshot signature is invalid or the file cannot be read. + * @throws PrunedTransactionException If we could not load previously pruned transactions (whilst they exist) */ - public Iota(IotaConfig configuration) throws TransactionPruningException, SnapshotException, SpentAddressesException { + public Iota(IotaConfig configuration) throws TransactionPruningException, SnapshotException, SpentAddressesException, PrunedTransactionException { this.configuration = configuration; // new refactored instances @@ -135,6 +142,9 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh transactionPruner = configuration.getLocalSnapshotsEnabled() && configuration.getLocalSnapshotsPruningEnabled() ? new AsyncTransactionPruner() : null; + + prunedTransactionProvider = transactionPruner != null ? new PrunedTransactionProviderImpl() : null; + transactionRequesterWorker = new TransactionRequesterWorkerImpl(); // legacy code @@ -196,7 +206,9 @@ public void init() throws Exception { } } - private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException { + private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException, + PrunedTransactionException { + //snapshot provider must be initialized first //because we check whether spent addresses data exists snapshotProvider.init(configuration); @@ -216,8 +228,14 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, spentAddressesService, bundleValidator); if (transactionPruner != null) { - transactionPruner.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, tipsViewModel, configuration); + if (prunedTransactionProvider != null) { + prunedTransactionProvider.init(configuration); + } + transactionPruner.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, + tipsViewModel, configuration, prunedTransactionProvider); } + + transactionRequesterWorker.init(tangle, transactionRequester, tipsViewModel, node); } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index ed35252223..9c19afee05 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -9,6 +9,7 @@ import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPruningException; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; import com.iota.iri.service.transactionpruning.jobs.MilestonePrunerJob; import com.iota.iri.service.transactionpruning.jobs.UnconfirmedSubtanglePrunerJob; import com.iota.iri.storage.Tangle; @@ -132,18 +133,21 @@ public class AsyncTransactionPruner implements TransactionPruner { * @param snapshotProvider data provider for the snapshots that are relevant for the node * @param tipsViewModel manager for the tips (required for removing pruned transactions from this manager) * @param config Configuration with important snapshot related configuration parameters + * @param prunedTransactionProvider manager which has logic about how to handle pruned transactions * @return the initialized instance itself to allow chaining */ public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvider, SpentAddressesService spentAddressesService, SpentAddressesProvider spentAddressesProvider, TipsViewModel tipsViewModel, - SnapshotConfig config) { + SnapshotConfig config, + PrunedTransactionProviderImpl prunedTransactionProvider) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.spentAddressesService = spentAddressesService; this.spentAddressesProvider = spentAddressesProvider; + this.prunedTransactionProvider = prunedTransactionProvider; this.tipsViewModel = tipsViewModel; this.config = config; From fcafae30879cf1afdeeda2af25097e6a6876a8eb Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Mon, 22 Apr 2019 15:26:11 +0200 Subject: [PATCH 13/20] Added prunedTransactionVerifier, which wraps the prunedTransactionProvider with additional checks --- src/main/java/com/iota/iri/Iota.java | 18 +- .../com/iota/iri/TransactionValidator.java | 15 +- src/main/java/com/iota/iri/network/Node.java | 25 ++- .../PrunedTransactionVerifier.java | 45 +++++ .../impl/PrunedTransactionVerifierImpl.java | 175 ++++++++++++++++++ 5 files changed, 270 insertions(+), 8 deletions(-) create mode 100644 src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java create mode 100644 src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 2c60c67306..1503e8b2cb 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -23,9 +23,11 @@ import com.iota.iri.service.tipselection.impl.*; import com.iota.iri.service.transactionpruning.PrunedTransactionException; import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; +import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionVerifierImpl; import com.iota.iri.storage.*; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Pair; @@ -99,6 +101,8 @@ public class Iota { public final TransactionRequesterWorkerImpl transactionRequesterWorker; public final PrunedTransactionProviderImpl prunedTransactionProvider; + + public final PrunedTransactionVerifier prunedTransactionVerifier; public final BundleValidator bundleValidator; @@ -114,6 +118,7 @@ public class Iota { public final TipSelector tipsSelector; + /** * Initializes the latest snapshot and then creates all services needed to run an IOTA node. * @@ -142,9 +147,7 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh transactionPruner = configuration.getLocalSnapshotsEnabled() && configuration.getLocalSnapshotsPruningEnabled() ? new AsyncTransactionPruner() : null; - - prunedTransactionProvider = transactionPruner != null ? new PrunedTransactionProviderImpl() : null; - + transactionRequesterWorker = new TransactionRequesterWorkerImpl(); // legacy code @@ -153,8 +156,13 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh tipsViewModel = new TipsViewModel(); transactionRequester = new TransactionRequester(tangle, snapshotProvider); transactionValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester); + + prunedTransactionProvider = transactionPruner != null ? new PrunedTransactionProviderImpl() : null; + prunedTransactionVerifier = transactionPruner != null ? new PrunedTransactionVerifierImpl( + prunedTransactionProvider, transactionRequester) : null; + node = new Node(tangle, snapshotProvider, transactionValidator, transactionRequester, tipsViewModel, - latestMilestoneTracker, configuration); + latestMilestoneTracker, configuration, prunedTransactionVerifier); replicator = new Replicator(node, configuration); udpReceiver = new UDPReceiver(node, configuration); tipsSolidifier = new TipsSolidifier(tangle, transactionValidator, tipsViewModel, configuration); @@ -185,7 +193,7 @@ public void init() throws Exception { tangle.clearMetadata(com.iota.iri.model.persistables.Transaction.class); } - transactionValidator.init(configuration.isTestnet(), configuration.getMwm()); + transactionValidator.init(configuration.isTestnet(), configuration.getMwm(), prunedTransactionVerifier); tipsSolidifier.init(); transactionRequester.init(configuration.getpRemoveRequest()); udpReceiver.init(); diff --git a/src/main/java/com/iota/iri/TransactionValidator.java b/src/main/java/com/iota/iri/TransactionValidator.java index 022e5c5d23..a797128cdf 100644 --- a/src/main/java/com/iota/iri/TransactionValidator.java +++ b/src/main/java/com/iota/iri/TransactionValidator.java @@ -10,6 +10,7 @@ import com.iota.iri.model.TransactionHash; import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier; import com.iota.iri.storage.Tangle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +53,8 @@ public class TransactionValidator { private final Set newSolidTransactionsOne = new LinkedHashSet<>(); private final Set newSolidTransactionsTwo = new LinkedHashSet<>(); + private PrunedTransactionVerifier prunedTransactionVerifier; + /** * Constructor for Tangle Validator * @@ -81,9 +84,12 @@ public class TransactionValidator { * regardless of parameter input. * @param mwm minimum weight magnitude: the minimal number of 9s that ought to appear at the end of the transaction * hash + * @param prunedTransactionProviderProvider for checking if a transaction is pruned previously */ - public void init(boolean testnet, int mwm) { + public void init(boolean testnet, int mwm, PrunedTransactionVerifier prunedTransactionVerifier) { setMwm(testnet, mwm); + + this.prunedTransactionVerifier = prunedTransactionVerifier; newSolidThread = new Thread(spawnSolidTransactionsPropagation(), "Solid TX cascader"); newSolidThread.start(); @@ -247,6 +253,13 @@ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTrans if(fromHash(tangle, hash).isSolid()) { return true; } + + // isPruned gets updated through parent CF verification. + // If one check is negative, isPossiblyPruned will return false. + if (this.prunedTransactionVerifier != null && this.prunedTransactionVerifier.isPossiblyPruned(hash)){ + return this.prunedTransactionVerifier.isPruned(hash); + } + Set analyzedHashes = new HashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); if(maxProcessedTransactions != Integer.MAX_VALUE) { maxProcessedTransactions += analyzedHashes.size(); diff --git a/src/main/java/com/iota/iri/network/Node.java b/src/main/java/com/iota/iri/network/Node.java index 9e72d545c1..3f9f760ee2 100644 --- a/src/main/java/com/iota/iri/network/Node.java +++ b/src/main/java/com/iota/iri/network/Node.java @@ -10,6 +10,8 @@ import com.iota.iri.model.TransactionHash; import com.iota.iri.service.milestone.LatestMilestoneTracker; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.transactionpruning.PrunedTransactionException; +import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier; import com.iota.iri.storage.Tangle; import net.openhft.hashing.LongHashFunction; import org.apache.commons.lang3.StringUtils; @@ -69,7 +71,8 @@ public class Node { private final TransactionValidator transactionValidator; private final LatestMilestoneTracker latestMilestoneTracker; private final TransactionRequester transactionRequester; - + private final PrunedTransactionVerifier prunedTransactionVerifier; + private static final SecureRandom rnd = new SecureRandom(); @@ -99,7 +102,10 @@ public class Node { * @param configuration Contains all the config. * */ - public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, final LatestMilestoneTracker latestMilestoneTracker, final NodeConfig configuration + public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, + final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, + final LatestMilestoneTracker latestMilestoneTracker, final NodeConfig configuration, + final PrunedTransactionVerifier prunedTransactionVerifier ) { this.configuration = configuration; this.tangle = tangle; @@ -112,6 +118,7 @@ public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final Transa int packetSize = configuration.getTransactionPacketSize(); this.sendingPacket = new DatagramPacket(new byte[packetSize], packetSize); this.tipRequestingPacket = new DatagramPacket(new byte[packetSize], packetSize); + this.prunedTransactionVerifier = prunedTransactionVerifier; } @@ -445,6 +452,20 @@ public void replyToRequestFromQueue() { public void processReceivedData(TransactionViewModel receivedTransactionViewModel, Neighbor neighbor) { boolean stored = false; + + try { + if (prunedTransactionVerifier != null && + prunedTransactionVerifier.waitingForHash(receivedTransactionViewModel.getHash())) { + + prunedTransactionVerifier.submitTransaction(receivedTransactionViewModel); + + // We dont store these old/pruned transactions + return; + } + } catch (PrunedTransactionException e) { + // We could not verify if this was pruned or not. Handle like normal + log.warn("Failed checking for pruned transaction state.", e); + } //store new transaction try { diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java new file mode 100644 index 0000000000..11c867358c --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java @@ -0,0 +1,45 @@ +package com.iota.iri.service.transactionpruning; + +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; + +public interface PrunedTransactionVerifier { + + /** + * Does a preliminary check to see if we should continue checking this hash its pruned status + * + * @param hash + * @return + * @throws PrunedTransactionException + */ + boolean isPossiblyPruned(Hash hash) throws PrunedTransactionException; + + /** + * Performs multiple checks on the hash to ensure it is pruned before + * + * @param hash + * @return + * @throws PrunedTransactionException + */ + boolean isPruned(Hash hash) throws PrunedTransactionException; + + /** + * Sends the transaction data for the verifier to use/store as it needs. + * {@link #waitingForHash(Hash)} should be used before to determine if we actually need it + * + * @param receivedTransactionViewModel + * @throws PrunedTransactionException + */ + void submitTransaction(TransactionViewModel receivedTransactionViewModel) throws PrunedTransactionException; + + /** + * Checks if we are waiting for this transaction its information. + * THis would mean that {@link #isPruned(Hash)} is called, and one of its parents has a reference to this hash. + * + * @param hash + * @return + * @throws PrunedTransactionException + */ + boolean waitingForHash(Hash hash) throws PrunedTransactionException; + +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java new file mode 100644 index 0000000000..9928392572 --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java @@ -0,0 +1,175 @@ +package com.iota.iri.service.transactionpruning.impl; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.transactionpruning.PrunedTransactionException; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; +import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier; + +public class PrunedTransactionVerifierImpl implements PrunedTransactionVerifier { + + private static final int PRUNED_CERTAIN = 10; + + private PrunedTransactionProvider provider; + + private TransactionRequester requester; + + /** + * List of Hashes who were possibly pruned, but turned out false after verifying + */ + private List verifiedFalse; + + /** + * List of children we have tested per main tx hash + */ + private Map> parents; + + /** + * Map of requested tx and our certainty it beeing pruned + */ + private Map prunedHashTest; + + + + public PrunedTransactionVerifierImpl(PrunedTransactionProvider provider, TransactionRequester requester) { + this.provider = provider; + this.requester = requester; + + verifiedFalse = new LinkedList<>(); + } + + /** + * Should be called before adding the transaction hash to ensure the initial hash is pruned + * + * @return + * @throws PrunedTransactionException + */ + @Override + public boolean isPossiblyPruned(Hash hash) throws PrunedTransactionException { + if (verifiedFalse.contains(hash)) { + return false; + } + return provider.containsTransaction(hash); + } + + /** + * + * {@inheritDoc} + */ + @Override + public boolean isPruned(Hash hash) throws PrunedTransactionException{ + if (verifiedFalse.contains(hash)) { + return false; + } + + if (prunedHashTest == null || !prunedHashTest.containsKey(hash)) { + return initializeVerify(hash); + } + + return prunedHashTest.get(hash) >= PRUNED_CERTAIN; + } + + /** + * + * {@inheritDoc} + */ + @Override + public void submitTransaction(TransactionViewModel receivedTransactionViewModel) throws PrunedTransactionException { + Hash parent = receivedTransactionViewModel.getHash(); + Hash child = getChildForParent(parent); + if (isPruned(child)) { + // We succeeded in the meantime. + return; + } + + if (isPossiblyPruned(parent)) { + // Add one to the map + prunedHashTest.merge(child, 1, Integer::sum); + + if (isPruned(child)) { + // We succeeded in the meantime. + clean(child); + return; + } + + List parents = addParentForChild(parent, child); + + // It could that they already got referenced through another tx + try { + if (!parents.contains(receivedTransactionViewModel.getBranchTransactionHash())){ + request(receivedTransactionViewModel.getBranchTransactionHash()); + } + if (!parents.contains(receivedTransactionViewModel.getTrunkTransactionHash())){ + request(receivedTransactionViewModel.getTrunkTransactionHash()); + } + } catch (Exception e) { + // We need to request but failed to do so + throw new PrunedTransactionException(e); + } + } else { + // False positive + clean(child); + verifiedFalse.add(child); + } + } + + /** + * + * {@inheritDoc} + */ + @Override + public boolean waitingForHash(Hash hash) { + return getChildForParent(hash) != null; + } + + private void clean(Hash child) { + prunedHashTest.remove(child); + parents.remove(child); + } + + private void request(Hash hash) throws Exception { + requester.requestTransaction(hash, false); + } + + private List addParentForChild(Hash parent, Hash child) { + if (parents == null) { + parents = new HashMap<>(); + } + + List list; + if (parents.containsKey(child)) { + list = parents.get(child); + } else { + list = parents.put(child, new LinkedList<>()); + } + + list.add(parent); + return list; + } + + private boolean initializeVerify(Hash hash) { + addParentForChild(hash, hash); + prunedHashTest.put(hash, 1); + return true; + } + + private Hash getChildForParent(Hash parent) { + if (parents == null) { + return null; + } + + for (Entry> entry : parents.entrySet()) { + if (entry.getValue().contains(parent)) { + return entry.getKey(); + } + } + + return null; + } +} From cbb23ca0b31f8b82189bc7ef3b2d02164420361c Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Mon, 22 Apr 2019 15:26:43 +0200 Subject: [PATCH 14/20] Added null params to tests (null checks are in the code) --- src/test/java/com/iota/iri/TransactionValidatorTest.java | 4 ++-- src/test/java/com/iota/iri/network/NodeTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java index 1ee6f751f6..336b3c5d4e 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/TransactionValidatorTest.java @@ -55,10 +55,10 @@ public static void tearDown() throws Exception { @Test public void testMinMwm() throws InterruptedException { - txValidator.init(false, 5); + txValidator.init(false, 5, null); assertTrue(txValidator.getMinWeightMagnitude() == 13); txValidator.shutdown(); - txValidator.init(false, MAINNET_MWM); + txValidator.init(false, MAINNET_MWM, null); } @Test diff --git a/src/test/java/com/iota/iri/network/NodeTest.java b/src/test/java/com/iota/iri/network/NodeTest.java index 0582463548..4b8467a94f 100644 --- a/src/test/java/com/iota/iri/network/NodeTest.java +++ b/src/test/java/com/iota/iri/network/NodeTest.java @@ -41,7 +41,7 @@ public void setUp() { // set up class under test nodeConfig = Mockito.mock(NodeConfig.class); - classUnderTest = new Node(null, null, null, null, null, null, nodeConfig); + classUnderTest = new Node(null, null, null, null, null, null, nodeConfig, null); // verify config calls in Node constructor verify(nodeConfig).getRequestHashSize(); From 41ed54f307723f6b192d0411a685fc7221033d85 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 24 Apr 2019 15:47:06 +0200 Subject: [PATCH 15/20] Added most basic tests, awaiting PR merge and code review --- .../impl/PrunedTransactionVerifierImpl.java | 11 ++- .../PrunedTransactionVerifierImplTest.java | 97 +++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java index 9928392572..31f8ac3cb7 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java @@ -42,6 +42,7 @@ public PrunedTransactionVerifierImpl(PrunedTransactionProvider provider, Transac this.requester = requester; verifiedFalse = new LinkedList<>(); + prunedHashTest = new HashMap<>(); } /** @@ -68,8 +69,8 @@ public boolean isPruned(Hash hash) throws PrunedTransactionException{ return false; } - if (prunedHashTest == null || !prunedHashTest.containsKey(hash)) { - return initializeVerify(hash); + if (!prunedHashTest.containsKey(hash)) { + initializeVerify(hash); } return prunedHashTest.get(hash) >= PRUNED_CERTAIN; @@ -146,17 +147,17 @@ private List addParentForChild(Hash parent, Hash child) { if (parents.containsKey(child)) { list = parents.get(child); } else { - list = parents.put(child, new LinkedList<>()); + list = new LinkedList<>(); + parents.put(child, list); } list.add(parent); return list; } - private boolean initializeVerify(Hash hash) { + private void initializeVerify(Hash hash) { addParentForChild(hash, hash); prunedHashTest.put(hash, 1); - return true; } private Hash getChildForParent(Hash parent) { diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java new file mode 100644 index 0000000000..e15e1c7379 --- /dev/null +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java @@ -0,0 +1,97 @@ +package com.iota.iri.service.transactionpruning; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import com.iota.iri.TransactionTestUtils; +import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.network.TransactionRequester; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionVerifierImpl; + +public class PrunedTransactionVerifierImplTest { + + private static final Hash A = TransactionTestUtils.getRandomTransactionHash(); + private static final Hash B = TransactionTestUtils.getRandomTransactionHash(); + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + public SnapshotConfig config; + + @Rule + public final TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule + public final TemporaryFolder logFolder = new TemporaryFolder(); + + @Mock + PrunedTransactionProviderImpl provider; + + @Mock + TransactionRequester requester; + + PrunedTransactionVerifierImpl verifier; + + @Before + public void setUp() throws PrunedTransactionException { + Mockito.when(config.getPrunedTransactionsDbPath()).thenReturn(dbFolder.getRoot().getAbsolutePath()); + Mockito.when(config.getPrunedTransactionsDbLogPath()).thenReturn(logFolder.getRoot().getAbsolutePath()); + + verifier = new PrunedTransactionVerifierImpl(provider, requester); + } + + @After + public void tearDown() { + dbFolder.delete(); + } + + @Test + public void isPossiblyPrunedTest() throws PrunedTransactionException { + Mockito.when(provider.containsTransaction(A)).thenReturn(true); + + assertTrue(verifier.isPossiblyPruned(A)); + assertFalse(verifier.isPossiblyPruned(B)); + } + + @Test + public void waitingForHashTest() throws PrunedTransactionException { + assertFalse(verifier.isPruned(A)); + assertTrue(verifier.waitingForHash(A)); + + assertFalse(verifier.waitingForHash(B)); + } + + @Test + public void completePruneCheckTest() throws PrunedTransactionException { + // 10 hashes in the pruner is enough to verify + // 0 -> 1 -> 3 -> 7 + // -> 4 -> 8 + // 2 -> 5 -> 9 + // -> 6 + + /* This makes no sense to do now, will continue after PR #1363 + TransactionViewModel[] tvms = new TransactionViewModel[10]; + tvms[9] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getRandomTransactionTrits()); + tvms[8] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getRandomTransactionTrits()); + tvms[7] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getRandomTransactionTrits()); + tvms[6] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getRandomTransactionTrits()); + + tvms[5] = TransactionTestUtils.createTransactionWithTrunkAndBranch(TransactionTestUtils., trunk, branch) + */ + } + +} From 306c5c759ffa4210ee73da98daf020dbde397b31 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 24 Apr 2019 16:18:49 +0200 Subject: [PATCH 16/20] Fixed nullpointer when receiving requested tx after pruned state confirmed --- .../impl/PrunedTransactionVerifierImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java index 31f8ac3cb7..7ea1b91c16 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java @@ -84,7 +84,7 @@ public boolean isPruned(Hash hash) throws PrunedTransactionException{ public void submitTransaction(TransactionViewModel receivedTransactionViewModel) throws PrunedTransactionException { Hash parent = receivedTransactionViewModel.getHash(); Hash child = getChildForParent(parent); - if (isPruned(child)) { + if (child == null || isPruned(child)) { // We succeeded in the meantime. return; } @@ -95,7 +95,7 @@ public void submitTransaction(TransactionViewModel receivedTransactionViewModel) if (isPruned(child)) { // We succeeded in the meantime. - clean(child); + parents.remove(child); return; } From 91d0a038ec668e9496b2e9eab3f3212a303a963d Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 25 Apr 2019 17:09:02 +0200 Subject: [PATCH 17/20] Remvoed imports, added assert messages --- src/main/java/com/iota/iri/Iota.java | 1 - .../transactionpruning/PrunedTransactionVerifier.java | 3 +++ .../PrunedTransactionVerifierImplTest.java | 11 +++++------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 1503e8b2cb..df8540cae7 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -22,7 +22,6 @@ import com.iota.iri.service.tipselection.*; import com.iota.iri.service.tipselection.impl.*; import com.iota.iri.service.transactionpruning.PrunedTransactionException; -import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java index 11c867358c..4cef983a2a 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifier.java @@ -3,6 +3,9 @@ import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +/** + * Verifies the integrity of a chain of pruned transactions by checking each trunk/branch. + */ public interface PrunedTransactionVerifier { /** diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java index e15e1c7379..6add733f76 100644 --- a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java @@ -15,7 +15,6 @@ import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.SnapshotConfig; -import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; @@ -63,16 +62,16 @@ public void tearDown() { public void isPossiblyPrunedTest() throws PrunedTransactionException { Mockito.when(provider.containsTransaction(A)).thenReturn(true); - assertTrue(verifier.isPossiblyPruned(A)); - assertFalse(verifier.isPossiblyPruned(B)); + assertTrue("A should be seen as pruned", verifier.isPossiblyPruned(A)); + assertFalse("B should not be seen as pruned", verifier.isPossiblyPruned(B)); } @Test public void waitingForHashTest() throws PrunedTransactionException { - assertFalse(verifier.isPruned(A)); - assertTrue(verifier.waitingForHash(A)); + assertFalse("A should not be seen as definitely pruned", verifier.isPruned(A)); + assertTrue("Verifier should be waiting for TVM for A", verifier.waitingForHash(A)); - assertFalse(verifier.waitingForHash(B)); + assertFalse("Verifier should not be waitingfor TVM for B", verifier.waitingForHash(B)); } @Test From 3ac580be3605fd307e2502fb05646a976f8651dd Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Tue, 14 May 2019 18:20:21 +0200 Subject: [PATCH 18/20] Fixed test method usage --- .../transactionpruning/PrunedTransactionVerifierImplTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java index 6add733f76..0572ac37e6 100644 --- a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java @@ -22,8 +22,8 @@ public class PrunedTransactionVerifierImplTest { - private static final Hash A = TransactionTestUtils.getRandomTransactionHash(); - private static final Hash B = TransactionTestUtils.getRandomTransactionHash(); + private static final Hash A = TransactionTestUtils.getTransactionHash(); + private static final Hash B = TransactionTestUtils.getTransactionHash(); @Rule public MockitoRule mockitoRule = MockitoJUnit.rule(); From 038a7419016cd6a5c0450da82ed8b9fb58b152e3 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 15 May 2019 18:05:13 +0200 Subject: [PATCH 19/20] Fixed test --- .../impl/PrunedTransactionVerifierImpl.java | 22 ++++++-- .../PrunedTransactionVerifierImplTest.java | 56 +++++++++++++++---- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java index 7ea1b91c16..71d3b40571 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java @@ -70,7 +70,11 @@ public boolean isPruned(Hash hash) throws PrunedTransactionException{ } if (!prunedHashTest.containsKey(hash)) { - initializeVerify(hash); + try { + initializeVerify(hash); + } catch (Exception e) { + throw new PrunedTransactionException("Failed to initialize pruned lookup", e); + } } return prunedHashTest.get(hash) >= PRUNED_CERTAIN; @@ -99,14 +103,16 @@ public void submitTransaction(TransactionViewModel receivedTransactionViewModel) return; } - List parents = addParentForChild(parent, child); + List parents = getParentsFor(child); // It could that they already got referenced through another tx try { if (!parents.contains(receivedTransactionViewModel.getBranchTransactionHash())){ + parents.add(receivedTransactionViewModel.getBranchTransactionHash()); request(receivedTransactionViewModel.getBranchTransactionHash()); } if (!parents.contains(receivedTransactionViewModel.getTrunkTransactionHash())){ + parents.add(receivedTransactionViewModel.getTrunkTransactionHash()); request(receivedTransactionViewModel.getTrunkTransactionHash()); } } catch (Exception e) { @@ -139,6 +145,13 @@ private void request(Hash hash) throws Exception { } private List addParentForChild(Hash parent, Hash child) { + List list = getParentsFor(child); + + list.add(parent); + return list; + } + + private List getParentsFor(Hash child) { if (parents == null) { parents = new HashMap<>(); } @@ -150,14 +163,13 @@ private List addParentForChild(Hash parent, Hash child) { list = new LinkedList<>(); parents.put(child, list); } - - list.add(parent); return list; } - private void initializeVerify(Hash hash) { + private void initializeVerify(Hash hash) throws Exception { addParentForChild(hash, hash); prunedHashTest.put(hash, 1); + request(hash); } private Hash getChildForParent(Hash parent) { diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java index 0572ac37e6..0d035d5e70 100644 --- a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionVerifierImplTest.java @@ -3,6 +3,8 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -10,11 +12,14 @@ import org.junit.rules.TemporaryFolder; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; +import org.mockito.stubbing.Answer; import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.network.TransactionRequester; import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; @@ -47,8 +52,8 @@ public class PrunedTransactionVerifierImplTest { @Before public void setUp() throws PrunedTransactionException { - Mockito.when(config.getPrunedTransactionsDbPath()).thenReturn(dbFolder.getRoot().getAbsolutePath()); - Mockito.when(config.getPrunedTransactionsDbLogPath()).thenReturn(logFolder.getRoot().getAbsolutePath()); + when(config.getPrunedTransactionsDbPath()).thenReturn(dbFolder.getRoot().getAbsolutePath()); + when(config.getPrunedTransactionsDbLogPath()).thenReturn(logFolder.getRoot().getAbsolutePath()); verifier = new PrunedTransactionVerifierImpl(provider, requester); } @@ -60,7 +65,7 @@ public void tearDown() { @Test public void isPossiblyPrunedTest() throws PrunedTransactionException { - Mockito.when(provider.containsTransaction(A)).thenReturn(true); + when(provider.containsTransaction(A)).thenReturn(true); assertTrue("A should be seen as pruned", verifier.isPossiblyPruned(A)); assertFalse("B should not be seen as pruned", verifier.isPossiblyPruned(B)); @@ -75,22 +80,51 @@ public void waitingForHashTest() throws PrunedTransactionException { } @Test - public void completePruneCheckTest() throws PrunedTransactionException { + public void completePruneCheckTest() throws Exception { // 10 hashes in the pruner is enough to verify // 0 -> 1 -> 3 -> 7 // -> 4 -> 8 // 2 -> 5 -> 9 // -> 6 - /* This makes no sense to do now, will continue after PR #1363 TransactionViewModel[] tvms = new TransactionViewModel[10]; - tvms[9] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getRandomTransactionTrits()); - tvms[8] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getRandomTransactionTrits()); - tvms[7] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getRandomTransactionTrits()); - tvms[6] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getRandomTransactionTrits()); + tvms[9] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getTransactionTrits()); + tvms[8] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getTransactionTrits()); + tvms[7] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getTransactionTrits()); + tvms[6] = TransactionTestUtils.createTransactionFromTrits(TransactionTestUtils.getTransactionTrits()); - tvms[5] = TransactionTestUtils.createTransactionWithTrunkAndBranch(TransactionTestUtils., trunk, branch) - */ + tvms[5] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[9].getHash(), tvms[9].getHash())); + tvms[4] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[8].getHash(), tvms[8].getHash())); + tvms[3] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[7].getHash(), tvms[7].getHash())); + + tvms[2] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[5].getHash(), tvms[6].getHash())); + tvms[1] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[3].getHash(), tvms[4].getHash())); + + tvms[0] = c(TransactionTestUtils.getTransactionTritsWithTrunkAndBranch(tvms[1].getHash(), tvms[2].getHash())); + + // Mock network request/response on the transactions + for (TransactionViewModel tvm : tvms) { + Mockito.doAnswer(new Answer() { + public Void answer(InvocationOnMock invocation) { + try { + verifier.submitTransaction(tvm); + } catch (PrunedTransactionException e) { + e.printStackTrace(); + } + return null; + } + }).when(requester).requestTransaction(tvm.getHash(), false); + + // Mock as pruned + when(provider.containsTransaction(tvm.getHash())).thenReturn(true); + } + + // Requests are handled directly (DFS), so it should mark pruned! + assertTrue("After checking 10 transactions, hash should be marked as pruned", + verifier.isPruned(tvms[0].getHash())); } + private TransactionViewModel c(byte[] trits) { + return TransactionTestUtils.createTransactionFromTrits(trits); + } } From cf97da8f23d00a0f184b437300e1036967e9659b Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 15 May 2019 18:24:25 +0200 Subject: [PATCH 20/20] Added comments --- .../impl/PrunedTransactionVerifierImpl.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java index 71d3b40571..15379e7c6e 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionVerifierImpl.java @@ -12,6 +12,11 @@ import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier; +/** + * Verifies the pruned state of a transaction hash by checking each parent the transaction references + * This is done until one transaction is found not to be pruned, or 10 transactions referencing this hash + * are found to be pruned. + */ public class PrunedTransactionVerifierImpl implements PrunedTransactionVerifier { private static final int PRUNED_CERTAIN = 10; @@ -31,12 +36,16 @@ public class PrunedTransactionVerifierImpl implements PrunedTransactionVerifier private Map> parents; /** - * Map of requested tx and our certainty it beeing pruned + * Map of requested tx and our certainty it being pruned */ private Map prunedHashTest; - - + /** + * Creates a pruned transaction verifier + * + * @param provider The provider we use to check for a pruned transaction + * @param requester Used to request transaction parents of a hash for verifying the pruned state + */ public PrunedTransactionVerifierImpl(PrunedTransactionProvider provider, TransactionRequester requester) { this.provider = provider; this.requester = requester; @@ -48,8 +57,8 @@ public PrunedTransactionVerifierImpl(PrunedTransactionProvider provider, Transac /** * Should be called before adding the transaction hash to ensure the initial hash is pruned * - * @return - * @throws PrunedTransactionException + * @return true if it could be pruned, false if it definitely wasn't pruned + * @throws PrunedTransactionException If the provider fails to check the transaction */ @Override public boolean isPossiblyPruned(Hash hash) throws PrunedTransactionException { @@ -89,7 +98,7 @@ public void submitTransaction(TransactionViewModel receivedTransactionViewModel) Hash parent = receivedTransactionViewModel.getHash(); Hash child = getChildForParent(parent); if (child == null || isPruned(child)) { - // We succeeded in the meantime. + // We succeeded in the meantime or we were not waiting for this at all return; }