From d688d9c6ddceb3a9a75d67d3ffff239e4574a59c Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Mon, 1 Apr 2019 02:08:06 +0200 Subject: [PATCH 01/11] Added cuckoofilter implementation details for looading from database --- .../com/iota/iri/conf/BaseIotaConfig.java | 36 ++++++- .../com/iota/iri/conf/SnapshotConfig.java | 13 +++ .../iri/model/persistables/CuckooBucket.java | 47 ++++++++++ .../impl/SpentAddressesProviderImpl.java | 3 +- .../PrunedTransactionException.java | 38 ++++++++ .../PrunedTransactionProvider.java | 36 +++++++ .../impl/PrunedTransactionProviderImpl.java | 93 +++++++++++++++++++ .../jobs/MilestonePrunerJob.java | 4 + .../jobs/UnconfirmedSubtanglePrunerJob.java | 1 - .../iri/utils/datastructure/CuckooFilter.java | 11 +++ .../datastructure/impl/CuckooFilterImpl.java | 20 ++++ 11 files changed, 294 insertions(+), 8 deletions(-) create mode 100644 src/main/java/com/iota/iri/model/persistables/CuckooBucket.java create mode 100644 src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java create mode 100644 src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java create mode 100644 src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index 3756f622a8..cd7efdf078 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -5,7 +5,6 @@ import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.iota.iri.IRI; import com.iota.iri.crypto.SpongeFactory; import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; @@ -109,7 +108,9 @@ public abstract class BaseIotaConfig implements IotaConfig { protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH; protected String spentAddressesDbPath = Defaults.SPENT_ADDRESSES_DB_PATH; protected String spentAddressesDbLogPath = Defaults.SPENT_ADDRESSES_DB_LOG_PATH; - + protected String prunedTransactionsDbPath = Defaults.PRUNED_TRANSACTIONS_DB_PATH; + protected String prunedTransactionsDbLogPath = Defaults.PRUNED_TRANSACTIONS_DB_LOG_PATH; + public BaseIotaConfig() { //empty constructor } @@ -650,13 +651,35 @@ protected void setSpentAddressesDbPath(String spentAddressesDbPath) { public String getSpentAddressesDbLogPath() { return spentAddressesDbLogPath; } - + @JsonProperty @Parameter(names = {"--spent-addresses-db-log-path"}, description = SnapshotConfig.Descriptions.SPENT_ADDRESSES_DB_LOG_PATH) - protected void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) { + protected void setPrunedTransactionDbLogPath(String spentAddressesDbLogPath) { this.spentAddressesDbLogPath = spentAddressesDbLogPath; } - + + @Override + public String getPrunedTransactionsDbLogPath() { + return prunedTransactionsDbLogPath; + } + + @JsonProperty + @Parameter(names = {"--pruned-transactions-db-log-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_LOG_PATH) + protected void setSpentAddressesDbLogPath(String prunedTransactionsDbLogPath) { + this.prunedTransactionsDbLogPath = prunedTransactionsDbLogPath; + } + + @Override + public String getPrunedTransactionsDbPath() { + return prunedTransactionsDbPath; + } + + @JsonProperty + @Parameter(names = {"--pruned-transactions-db-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_PATH) + protected void setPrunedTransactionsDbPath(String prunedTransactionsDbPath) { + this.prunedTransactionsDbPath = prunedTransactionsDbPath; + } + /** * Checks if ZMQ is enabled. * @return true if zmqEnableTcp or zmqEnableIpc is set. @@ -933,6 +956,9 @@ public interface Defaults { int LOCAL_SNAPSHOTS_DEPTH_MIN = 100; String SPENT_ADDRESSES_DB_PATH = "spent-addresses-db"; String SPENT_ADDRESSES_DB_LOG_PATH = "spent-addresses-log"; + + String PRUNED_TRANSACTIONS_DB_LOG_PATH = "spent-addresses-db"; + String PRUNED_TRANSACTIONS_DB_PATH = "spent-addresses-log"; String LOCAL_SNAPSHOTS_BASE_PATH = "mainnet"; String SNAPSHOT_FILE = "/snapshotMainnet.txt"; diff --git a/src/main/java/com/iota/iri/conf/SnapshotConfig.java b/src/main/java/com/iota/iri/conf/SnapshotConfig.java index 33c30fb705..b213c4f8b6 100644 --- a/src/main/java/com/iota/iri/conf/SnapshotConfig.java +++ b/src/main/java/com/iota/iri/conf/SnapshotConfig.java @@ -76,6 +76,16 @@ public interface SnapshotConfig extends Config { */ String getSpentAddressesDbLogPath(); + /** + * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_PATH} + */ + String getPrunedTransactionsDbPath(); + + /** + * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_LOG_PATH} + */ + String getPrunedTransactionsDbLogPath(); + interface Descriptions { String LOCAL_SNAPSHOTS_ENABLED = "Flag that determines if local snapshots are enabled."; @@ -94,5 +104,8 @@ interface Descriptions { "from previous epochs"; String SPENT_ADDRESSES_DB_PATH = "The folder where the spent addresses DB saves its data."; String SPENT_ADDRESSES_DB_LOG_PATH = "The folder where the spent addresses DB saves its logs."; + String PRUNED_TRANSACTIONS_DB_PATH = "The folder where the pruned transactions DB saves its data."; + String PRUNED_TRANSACTIONS_DB_LOG_PATH = "The folder where the pruned transactions DB saves its logs."; } + } diff --git a/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java b/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java new file mode 100644 index 0000000000..3b5b847620 --- /dev/null +++ b/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java @@ -0,0 +1,47 @@ +package com.iota.iri.model.persistables; + +import java.util.BitSet; + +import org.apache.commons.lang3.ArrayUtils; + +import com.iota.iri.model.IntegerIndex; +import com.iota.iri.storage.Persistable; +import com.iota.iri.utils.Serializer; + +public class CuckooBucket implements Persistable { + + public BitSet bucketBits; + public IntegerIndex bucketIndex; + + @Override + public byte[] bytes() { + return ArrayUtils.addAll(bucketIndex.bytes(), bucketBits.toByteArray()); + } + + @Override + public void read(byte[] bytes) { + if(bytes != null) { + bucketIndex = new IntegerIndex(Serializer.getInteger(bytes)); + + bucketBits = new BitSet(bytes.length - 2); + for (int i = 1; i < bytes.length; i++) { + bucketBits.set(i-1, bytes[i]); + } + } + } + + @Override + public byte[] metadata() { + return new byte[0]; + } + + @Override + public void readMetadata(byte[] bytes) { + } + + @Override + public boolean merge() { + return false; + } + +} diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index c59c61dc6e..8bcc7503c7 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -54,8 +54,7 @@ public SpentAddressesProviderImpl init(SnapshotConfig config) {{put("spent-addresses", SpentAddress.class);}}, null); this.rocksDBPersistenceProvider.init(); readPreviousEpochsSpentAddresses(); - } - catch (Exception e) { + } catch (Exception e) { throw new SpentAddressesException("There is a problem with accessing stored spent addresses", e); } return this; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java new file mode 100644 index 0000000000..4e1ea756da --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java @@ -0,0 +1,38 @@ +package com.iota.iri.service.transactionpruning; + +/** + * This class is used to wrap exceptions that are specific to the pruned transaction persistance. + * + * It allows us to distinct between the different kinds of errors that can happen during the execution of the code. + */ +public class PrunedTransactionException extends Exception { + /** + * Constructor of the exception which allows us to provide a specific error message and the cause of the error. + * + * @param message reason why this error occurred + * @param cause wrapped exception that caused this error + */ + public PrunedTransactionException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor of the exception which allows us to provide a specific error message without having an underlying + * cause. + * + * @param message reason why this error occurred + */ + public PrunedTransactionException(String message) { + super(message); + } + + /** + * Constructor of the exception which allows us to wrap the underlying cause of the error without providing a + * specific reason. + * + * @param cause wrapped exception that caused this error + */ + public PrunedTransactionException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java new file mode 100644 index 0000000000..264ba34c9d --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java @@ -0,0 +1,36 @@ +package com.iota.iri.service.transactionpruning; + +import java.util.Collection; + +import com.iota.iri.model.Hash; + +/** + * Find, mark and store pruned transactions + */ +public interface PrunedTransactionProvider { + + /** + * Checks if this transactions has been pruned + * + * @param transactionHash The transaction to check for + * @return true if it is, else false + * @throws PrunedTransactionException If the provider fails to check the transaction + */ + boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException; + + /** + * Mark a transaction as spent. + * + * @param transactionHash the transaction which we want to mark. + * @throws PrunedTransactionException If the provider fails to add the transaction + */ + void addTransaction(Hash transactionHash) throws PrunedTransactionException; + + /** + * Mark all transactions as pruned. + * + * @param transactionHashes The transactions we want to mark + * @throws PrunedTransactionException If the provider fails to add a transaction + */ + void AddTransactionBatch(Collection transactionHashes) throws PrunedTransactionException; +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java new file mode 100644 index 0000000000..8c206b5bd1 --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -0,0 +1,93 @@ +package com.iota.iri.service.transactionpruning.impl; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.model.Hash; +import com.iota.iri.model.HashFactory; +import com.iota.iri.model.persistables.CuckooBucket; +import com.iota.iri.model.persistables.SpentAddress; +import com.iota.iri.service.spentaddresses.SpentAddressesException; +import com.iota.iri.service.spentaddresses.impl.SpentAddressesProviderImpl; +import com.iota.iri.service.transactionpruning.PrunedTransactionException; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; +import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; +import com.iota.iri.utils.datastructure.CuckooFilter; + +public class PrunedTransactionProviderImpl implements PrunedTransactionProvider { + + private static final Logger log = LoggerFactory.getLogger(PrunedTransactionProviderImpl.class); + + private RocksDBPersistenceProvider persistenceProvider; + + private CuckooFilter filter; + + private SnapshotConfig config; + + /** + * Starts the PrunedTransactionProvider by reading the current pruned transactions from a persistence provider + * + * @param config The snapshot configuration used for file location + * @return the current instance + * @throws SpentAddressesException if we failed to create a file at the designated location + */ + public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTransactionException { + this.config = config; + try { + this.persistenceProvider = new RocksDBPersistenceProvider( + config.getPrunedTransactionsDbPath(), + config.getPrunedTransactionsDbLogPath(), + 1000, + new HashMap>(1) + {{put("pruned-transactions", CuckooBucket.class);}}, null); + this.persistenceProvider.init(); + readPreviousPrunedTransactions(); + } + catch (Exception e) { + throw new PrunedTransactionException("There is a problem with accessing stored spent addresses", e); + } + + return this; + } + + private void readPreviousPrunedTransactions() throws PrunedTransactionException { + if (config.isTestnet()) { + return; + } + + try { + for (byte[] bucketData : persistenceProvider.loadAllKeysFromTable(CuckooBucket.class)) { + CuckooBucket bucket = new CuckooBucket(); + bucket.read(bucketData); + filter.update(bucket); + } + } catch (IllegalArgumentException e) { + throw new PrunedTransactionException(e); + } + } + + @Override + public boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException { + return false; + } + + @Override + public void addTransaction(Hash transactionHash) throws PrunedTransactionException { + + } + + @Override + public void AddTransactionBatch(Collection transactionHashes) throws PrunedTransactionException { + + } + +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index b182cc37c9..9838dd0840 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -233,6 +233,10 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { } catch (TransactionPruningException e) { throw new RuntimeException(e); } + } else { + // Confirmed TX, add to data structure + // either add to list, or add a new job to add to the cuckoofilter + } } else if(Milestone.class.equals(element.hi)) { MilestoneViewModel.clear(((IntegerIndex) element.low).getValue()); diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java index 780d078267..c29fec8994 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java @@ -4,7 +4,6 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; import com.iota.iri.model.persistables.Transaction; -import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.storage.Indexable; diff --git a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java index 9807ff2119..6391d521a0 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java +++ b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java @@ -1,5 +1,7 @@ package com.iota.iri.utils.datastructure; +import com.iota.iri.model.persistables.CuckooBucket; + /** * The Cuckoo Filter is a probabilistic data structure that supports fast set membership testing. * @@ -83,4 +85,13 @@ public interface CuckooFilter { * @return the amount of stored items */ int size(); + + /** + * Update a part of this filter with the bucket information supplied. + * Discards any previous data inside this bucket. + * + * @param bucket the bucket data we use to update the filter + * @throws IllegalArgumentException when the bucket does not contain data which is valid for this filter + */ + void update(CuckooBucket bucket) throws IllegalArgumentException ; } diff --git a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java index 8efa4e2aae..46a8d8e292 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java +++ b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java @@ -1,5 +1,6 @@ package com.iota.iri.utils.datastructure.impl; +import com.iota.iri.model.persistables.CuckooBucket; import com.iota.iri.utils.BitSetUtils; import com.iota.iri.utils.datastructure.CuckooFilter; @@ -137,6 +138,25 @@ public boolean add(String item) throws IndexOutOfBoundsException { public boolean add(byte[] item) throws IndexOutOfBoundsException { return add(new CuckooFilterItem(hashFunction.digest(item))); } + /** + * {@inheritDoc} + * + */ + @Override + public void update(CuckooBucket bucket) throws IllegalArgumentException { + int amountInBucket = bucketSize; + if (bucket.bucketBits.length() / amountInBucket > 0) { + + } + + for (int i=0; i < amountInBucket; i++) { + cuckooFilterTable.set(bucket.bucketIndex.getValue(), i, bucket.bucketBits.get( + fingerPrintSize * i, + fingerPrintSize * (i + 1) + )); + + } + } /** * {@inheritDoc} From 989aaba819e7cb0ac9d5d9fa5596c6c165a8d62f Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Fri, 12 Apr 2019 12:08:58 +0200 Subject: [PATCH 02/11] Added PrunedProvider, added it to prunerjobs, updated int parsing serializer --- .../iri/model/persistables/CuckooBucket.java | 48 +++++- .../PrunedTransactionProvider.java | 2 +- .../TransactionPrunerJob.java | 14 ++ .../async/AsyncTransactionPruner.java | 7 + .../impl/PrunedTransactionProviderImpl.java | 143 ++++++++++++++++-- .../jobs/AbstractTransactionPrunerJob.java | 24 ++- .../jobs/HashPruningJob.java | 19 +++ .../jobs/MilestonePrunerJob.java | 5 + .../java/com/iota/iri/utils/Serializer.java | 5 +- .../iri/utils/datastructure/CuckooFilter.java | 7 +- .../datastructure/impl/CuckooFilterImpl.java | 24 ++- .../PrunedTransactionProviderImplTest.java | 69 +++++++++ 12 files changed, 337 insertions(+), 30 deletions(-) create mode 100644 src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java create mode 100644 src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java diff --git a/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java b/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java index 3b5b847620..10f0600f68 100644 --- a/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java +++ b/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java @@ -8,37 +8,75 @@ import com.iota.iri.storage.Persistable; import com.iota.iri.utils.Serializer; +/** + * Persistable to manage the data we get as a result of pruning a milestone and its transactions + */ public class CuckooBucket implements Persistable { + /** + * The filter number it belonged to in previous cycles + */ + public IntegerIndex bucketId; + + /** + * + */ public BitSet bucketBits; public IntegerIndex bucketIndex; + /** + * + * {@inheritDoc} + */ @Override public byte[] bytes() { - return ArrayUtils.addAll(bucketIndex.bytes(), bucketBits.toByteArray()); + byte[] index = bucketIndex.bytes(); + byte[] num = bucketId.bytes(); + return ArrayUtils.addAll(ArrayUtils.addAll(num, index), bucketBits.toByteArray()); } + /** + * Reads a CuckooBucket from the provided bytes. + * First 4 bytes are the bucket id, second 4 are the index inside that bucket + * Rest of the bytes is the bucket data + * + * {@inheritDoc} + */ @Override public void read(byte[] bytes) { if(bytes != null) { - bucketIndex = new IntegerIndex(Serializer.getInteger(bytes)); + bucketId = new IntegerIndex(Serializer.getInteger(bytes, 0)); + bucketIndex = new IntegerIndex(Serializer.getInteger(bytes, 4)); - bucketBits = new BitSet(bytes.length - 2); - for (int i = 1; i < bytes.length; i++) { - bucketBits.set(i-1, bytes[i]); + short start = 8; + bucketBits = new BitSet(bytes.length - start); + for (int i = start; i < bytes.length; i++) { + bucketBits.set(i-start, bytes[i]); } } } + /** + * + * {@inheritDoc} + */ @Override public byte[] metadata() { return new byte[0]; } + /** + * + * {@inheritDoc} + */ @Override public void readMetadata(byte[] bytes) { } + /** + * + * {@inheritDoc} + */ @Override public boolean merge() { return false; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java index 264ba34c9d..ea449b2f22 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java @@ -32,5 +32,5 @@ public interface PrunedTransactionProvider { * @param transactionHashes The transactions we want to mark * @throws PrunedTransactionException If the provider fails to add a transaction */ - void AddTransactionBatch(Collection transactionHashes) throws PrunedTransactionException; + void addTransactionBatch(Collection transactionHashes) throws PrunedTransactionException; } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java index 5398fcf4a5..7d0797405b 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java @@ -112,6 +112,20 @@ public interface TransactionPrunerJob { * @param transactionPrunerJobStatus new execution status of the job */ void setStatus(TransactionPrunerJobStatus transactionPrunerJobStatus); + + /** + * Getter for the pruned transaction maintainer. + * + * @return pruned transaction maintainer of the job. + */ + PrunedTransactionProvider getPrunedProvider(); + + /** + * Setter for the pruned transaction maintainer. + * + * @param prunedTransactionProvider pruned transaction maintainer of the job + */ + void setPrunedProvider(PrunedTransactionProvider prunedTransactionProvider); /** * This method processes the cleanup job and performs the actual pruning. diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index 796afa0c57..99a47edc62 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -4,6 +4,7 @@ import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPruningException; @@ -104,6 +105,11 @@ public class AsyncTransactionPruner implements TransactionPruner { */ private final Map, JobQueue> jobQueues = new HashMap<>(); + /** + * + */ + private PrunedTransactionProvider prunedTransactionProvider; + /** * This method initializes the instance and registers its dependencies.
*
@@ -149,6 +155,7 @@ public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvi @Override public void addJob(TransactionPrunerJob job) throws TransactionPruningException { job.setTransactionPruner(this); + job.setPrunedProvider(prunedTransactionProvider); job.setSpentAddressesService(spentAddressesService); job.setTangle(tangle); job.setTipsViewModel(tipsViewModel); diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java index 8c206b5bd1..10316f9b40 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -1,37 +1,66 @@ package com.iota.iri.service.transactionpruning.impl; -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import org.apache.commons.collections4.queue.CircularFifoQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.iota.iri.conf.SnapshotConfig; import com.iota.iri.model.Hash; -import com.iota.iri.model.HashFactory; import com.iota.iri.model.persistables.CuckooBucket; -import com.iota.iri.model.persistables.SpentAddress; import com.iota.iri.service.spentaddresses.SpentAddressesException; -import com.iota.iri.service.spentaddresses.impl.SpentAddressesProviderImpl; import com.iota.iri.service.transactionpruning.PrunedTransactionException; import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.datastructure.CuckooFilter; +import com.iota.iri.utils.datastructure.impl.CuckooFilterImpl; +/** + * + */ public class PrunedTransactionProviderImpl implements PrunedTransactionProvider { private static final Logger log = LoggerFactory.getLogger(PrunedTransactionProviderImpl.class); - + + private static final int FILTER_SIZE = 200000; + + private static final int MAX_FILTERS = 10; + private RocksDBPersistenceProvider persistenceProvider; + + private SnapshotConfig config; - private CuckooFilter filter; + private CircularFifoQueue filters; + private CuckooFilter lastAddedFilter; - private SnapshotConfig config; + // Once this exceeds integer max range, this will give problems. + private Integer highestIndex = -1; + + private int filterSize; + + /** + * + */ + public PrunedTransactionProviderImpl() { + this(FILTER_SIZE); + } + + /** + * + * @param filterSize + */ + public PrunedTransactionProviderImpl(int filterSize) { + this.filterSize = filterSize; + } /** * Starts the PrunedTransactionProvider by reading the current pruned transactions from a persistence provider @@ -50,10 +79,14 @@ public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTr new HashMap>(1) {{put("pruned-transactions", CuckooBucket.class);}}, null); this.persistenceProvider.init(); + + // 10 * 1.000.000 * 4 hashes we can hold, with a total of 80mb size when max filled. + // The database has overhead for the bucket index, and allows a maximum of 2^7 - 1 filters + filters = new CircularFifoQueue(MAX_FILTERS); + readPreviousPrunedTransactions(); - } - catch (Exception e) { - throw new PrunedTransactionException("There is a problem with accessing stored spent addresses", e); + } catch (Exception e) { + throw new PrunedTransactionException("There is a problem with accessing previously pruned transactions", e); } return this; @@ -61,33 +94,111 @@ public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTr private void readPreviousPrunedTransactions() throws PrunedTransactionException { if (config.isTestnet()) { + newFilter(); return; } try { - for (byte[] bucketData : persistenceProvider.loadAllKeysFromTable(CuckooBucket.class)) { + + TreeMap filters = new TreeMap<>(); + + // Load all data from all filters + List bytes = persistenceProvider.loadAllKeysFromTable(CuckooBucket.class); + for (byte[] bucketData : bytes) { CuckooBucket bucket = new CuckooBucket(); bucket.read(bucketData); - filter.update(bucket); + + if (MAX_FILTERS < bucket.bucketId.getValue()) { + throw new PrunedTransactionException("Database contains more filters then we can store"); + } + + // Find its bucket, or create it if wasnt there + CuckooFilter filter = filters.get(bucket.bucketId.getValue()); + if (null == filter) { + filter = new CuckooFilterImpl(filterSize, 4, 16); + filters.put(bucket.bucketId.getValue(), filter); + } + + // Don't update using the entire CuckooBucket so that packages are separate-able + filter.update(bucket.bucketIndex.getValue(), bucket.bucketBits); + } + + //Then add all in order, treemap maintains order from lowest to highest key + for (CuckooFilter filter : filters.values()) { + if (null != filter) { + this.filters.add(filter); + } + } + + Entry entry = filters.lastEntry(); + if (null != entry) { + lastAddedFilter = entry.getValue(); + highestIndex = entry.getKey(); + } else { + newFilter(); } + } catch (IllegalArgumentException e) { throw new PrunedTransactionException(e); } } + + private void persistFilter(CuckooFilter filter, Integer index) { + + } @Override public boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException { + byte[] hashBytes = transactionHash.bytes(); + + // last filter added is most recent, thus most likely to be requested + CuckooFilter[] filterArray = filters.toArray(new CuckooFilter[filters.size()]); + for (int i = filterArray.length - 1; i >=0; i--) { + if (filterArray[i].contains(hashBytes)) { + return true; + } + } return false; } + /** + * + * {@inheritDoc} + */ @Override public void addTransaction(Hash transactionHash) throws PrunedTransactionException { + if (null == lastAddedFilter) { + newFilter(); + } + if (!lastAddedFilter.add(transactionHash.bytes())){ + newFilter().add(transactionHash.bytes()); + } } + /** + * + * {@inheritDoc} + */ @Override - public void AddTransactionBatch(Collection transactionHashes) throws PrunedTransactionException { - + public void addTransactionBatch(Collection transactionHashes) throws PrunedTransactionException { + // Should we create a new filter when we get this method call? + // It probably implies a new pruned job completed, and these TX would be checked more often then the older ones. + for (Hash transactionHash : transactionHashes) { + addTransaction(transactionHash); + } + persistFilter(lastAddedFilter, highestIndex); } + private CuckooFilter newFilter() { + if (filters.isAtFullCapacity()) { + log.debug("Removing " + filters.peek()); + } + + highestIndex++; + + // We keep a reference to the last filter to prevent looking it up every time + filters.offer(lastAddedFilter = new CuckooFilterImpl(filterSize, 4, 16)); + return lastAddedFilter; + } } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java index ebaf0744df..bf87215f9d 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java @@ -3,6 +3,7 @@ import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; @@ -43,6 +44,11 @@ public abstract class AbstractTransactionPrunerJob implements TransactionPrunerJ */ private Snapshot snapshot; + /** + * Holds a reference to the provider of pruned transactions, which we will use for speeding up old references checks + */ + private PrunedTransactionProvider prunedTransactionProvider; + /** * {@inheritDoc} */ @@ -127,7 +133,23 @@ public TransactionPrunerJobStatus getStatus() { public void setStatus(TransactionPrunerJobStatus status) { this.status = status; } - + + /** + * {@inheritDoc} + */ + @Override + public PrunedTransactionProvider getPrunedProvider() { + return prunedTransactionProvider; + } + + /** + * {@inheritDoc} + */ + @Override + public void setPrunedProvider(PrunedTransactionProvider prunedTransactionProvider) { + this.prunedTransactionProvider = prunedTransactionProvider; + } + /** * {@inheritDoc} */ diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java new file mode 100644 index 0000000000..7a0d6a4fc8 --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java @@ -0,0 +1,19 @@ +package com.iota.iri.service.transactionpruning.jobs; + +import com.iota.iri.service.transactionpruning.TransactionPruningException; + +public class HashPruningJob extends AbstractTransactionPrunerJob { + + public HashPruningJob() { + + } + + @Override + public void process() throws TransactionPruningException { + } + + @Override + public String serialize() { + return null; + } +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index 9838dd0840..62d25fe816 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -244,6 +245,10 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { }); getTangle().deleteBatch(elementsToDelete); + getPrunedProvider().addTransactionBatch(elementsToDelete + .stream() + .map(a -> (Hash) a.low) + .collect(Collectors.toList())); } catch(Exception e) { throw new TransactionPruningException("failed to cleanup milestone #" + getCurrentIndex(), e); } diff --git a/src/main/java/com/iota/iri/utils/Serializer.java b/src/main/java/com/iota/iri/utils/Serializer.java index da14da5380..f46e43edb5 100644 --- a/src/main/java/com/iota/iri/utils/Serializer.java +++ b/src/main/java/com/iota/iri/utils/Serializer.java @@ -41,10 +41,13 @@ public static int getInteger(byte[] bytes) { return getInteger(bytes, 0); } public static int getInteger(byte[] bytes, int start) { + return getInteger(bytes, start, Integer.BYTES); + } + + public static int getInteger(byte[] bytes, int start, int length) { if(bytes == null) { return 0; } - int length = Integer.BYTES; int res = 0; for (int i=0; i< length;i++) { res |= (bytes[start + i] & 0xFFL) << ((length-i-1) * 8); diff --git a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java index 6391d521a0..8cabd828ef 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java +++ b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java @@ -1,5 +1,7 @@ package com.iota.iri.utils.datastructure; +import java.util.BitSet; + import com.iota.iri.model.persistables.CuckooBucket; /** @@ -90,8 +92,9 @@ public interface CuckooFilter { * Update a part of this filter with the bucket information supplied. * Discards any previous data inside this bucket. * - * @param bucket the bucket data we use to update the filter + * @param index The index of the bucket we are updating + * @param bits the bucket data we use to update the filter * @throws IllegalArgumentException when the bucket does not contain data which is valid for this filter */ - void update(CuckooBucket bucket) throws IllegalArgumentException ; + void update(int index, BitSet bits) throws IllegalArgumentException ; } diff --git a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java index 46a8d8e292..6d9f75b601 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java +++ b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java @@ -12,6 +12,11 @@ * This class implements the basic contract of the {@link CuckooFilter}. */ public class CuckooFilterImpl implements CuckooFilter { + + private static int CUR_INDEX = 0; + + int index = CUR_INDEX ++; + /** * The amount of times we try to kick elements when inserting before we consider the index to be too full. */ @@ -143,14 +148,20 @@ public boolean add(byte[] item) throws IndexOutOfBoundsException { * */ @Override - public void update(CuckooBucket bucket) throws IllegalArgumentException { + public void update(int index, BitSet bits) throws IllegalArgumentException { int amountInBucket = bucketSize; - if (bucket.bucketBits.length() / amountInBucket > 0) { - + if (bits.length() % fingerPrintSize != 0) { + throw new IllegalArgumentException("Provided bits do not match fingerprint scheme"); + } else if (bits.length() % fingerPrintSize > amountInBucket) { + throw new IllegalArgumentException("Provided fingerprint data will overflow the bucket"); + } else if (index > tableSize * 0.955) { + throw new IllegalArgumentException("Provided bucket exceeds filter size"); + } else if (false) { + // Can we recover input in any case when expected input does not match given? } for (int i=0; i < amountInBucket; i++) { - cuckooFilterTable.set(bucket.bucketIndex.getValue(), i, bucket.bucketBits.get( + cuckooFilterTable.set(index, i, bits.get( fingerPrintSize * i, fingerPrintSize * (i + 1) )); @@ -606,4 +617,9 @@ public CuckooFilterTable delete(int bucketIndex, int slotIndex) { return set(bucketIndex, slotIndex, null); } } + + @Override + public String toString() { + return index +" "; + } } diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java new file mode 100644 index 0000000000..e609c4654d --- /dev/null +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java @@ -0,0 +1,69 @@ +package com.iota.iri.service.transactionpruning; + +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import com.iota.iri.TransactionTestUtils; +import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.model.Hash; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; + +public class PrunedTransactionProviderImplTest { + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + public SnapshotConfig config; + + @Rule + public final TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule + public final TemporaryFolder logFolder = new TemporaryFolder(); + + PrunedTransactionProviderImpl provider; + + @Before + public void setUp() throws PrunedTransactionException { + Mockito.when(config.getPrunedTransactionsDbPath()).thenReturn(dbFolder.getRoot().getAbsolutePath()); + Mockito.when(config.getPrunedTransactionsDbLogPath()).thenReturn(logFolder.getRoot().getAbsolutePath()); + + // Filter size of 10, for easier testing + provider = new PrunedTransactionProviderImpl(10); + provider.init(config); + } + + @After + public void tearDown() { + dbFolder.delete(); + } + + @Test + public void test() throws PrunedTransactionException { + Hash randomHash = TransactionTestUtils.getRandomTransactionHash(); + provider.addTransaction(randomHash); + for (int i=0; i<100; i++) { + provider.addTransaction(TransactionTestUtils.getRandomTransactionHash()); + } + + int contains = 0; + for (int i=0; i<100; i++) { + if (provider.containsTransaction(randomHash)){ + contains++; + } + } + + System.out.println(contains); + assertTrue("Provider should contain the hash", contains > 90); + } +} From df1a2ddd59dc0bf8a5b995a2862963f482fb5c18 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Sun, 14 Apr 2019 19:58:41 +0200 Subject: [PATCH 03/11] Added saving filters --- .../{CuckooBucket.java => Cuckoo.java} | 21 ++--- .../impl/PrunedTransactionProviderImpl.java | 91 ++++++++++++------- .../iri/utils/datastructure/CuckooFilter.java | 12 +-- .../datastructure/impl/CuckooFilterImpl.java | 73 ++++++++------- 4 files changed, 115 insertions(+), 82 deletions(-) rename src/main/java/com/iota/iri/model/persistables/{CuckooBucket.java => Cuckoo.java} (68%) diff --git a/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java similarity index 68% rename from src/main/java/com/iota/iri/model/persistables/CuckooBucket.java rename to src/main/java/com/iota/iri/model/persistables/Cuckoo.java index 10f0600f68..780f18bfa4 100644 --- a/src/main/java/com/iota/iri/model/persistables/CuckooBucket.java +++ b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java @@ -11,18 +11,17 @@ /** * Persistable to manage the data we get as a result of pruning a milestone and its transactions */ -public class CuckooBucket implements Persistable { +public class Cuckoo implements Persistable { /** * The filter number it belonged to in previous cycles */ - public IntegerIndex bucketId; + public IntegerIndex filterId; /** * */ - public BitSet bucketBits; - public IntegerIndex bucketIndex; + public BitSet filterBits; /** * @@ -30,9 +29,8 @@ public class CuckooBucket implements Persistable { */ @Override public byte[] bytes() { - byte[] index = bucketIndex.bytes(); - byte[] num = bucketId.bytes(); - return ArrayUtils.addAll(ArrayUtils.addAll(num, index), bucketBits.toByteArray()); + byte[] num = filterId.bytes(); + return ArrayUtils.addAll(num, filterBits.toByteArray()); } /** @@ -45,13 +43,12 @@ public byte[] bytes() { @Override public void read(byte[] bytes) { if(bytes != null) { - bucketId = new IntegerIndex(Serializer.getInteger(bytes, 0)); - bucketIndex = new IntegerIndex(Serializer.getInteger(bytes, 4)); + filterId = new IntegerIndex(Serializer.getInteger(bytes, 0)); - short start = 8; - bucketBits = new BitSet(bytes.length - start); + short start = 4; + filterBits = new BitSet(bytes.length - start); for (int i = start; i < bytes.length; i++) { - bucketBits.set(i-start, bytes[i]); + filterBits.set(i-start, bytes[i]); } } } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java index 10316f9b40..aa3087d357 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -15,10 +15,12 @@ import com.iota.iri.conf.SnapshotConfig; import com.iota.iri.model.Hash; -import com.iota.iri.model.persistables.CuckooBucket; +import com.iota.iri.model.IntegerIndex; +import com.iota.iri.model.persistables.Cuckoo; import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.transactionpruning.PrunedTransactionException; import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; +import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.datastructure.CuckooFilter; @@ -77,11 +79,9 @@ public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTr config.getPrunedTransactionsDbLogPath(), 1000, new HashMap>(1) - {{put("pruned-transactions", CuckooBucket.class);}}, null); + {{put("pruned-transactions", Cuckoo.class);}}, null); this.persistenceProvider.init(); - // 10 * 1.000.000 * 4 hashes we can hold, with a total of 80mb size when max filled. - // The database has overhead for the bucket index, and allows a maximum of 2^7 - 1 filters filters = new CircularFifoQueue(MAX_FILTERS); readPreviousPrunedTransactions(); @@ -94,33 +94,30 @@ public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTr private void readPreviousPrunedTransactions() throws PrunedTransactionException { if (config.isTestnet()) { - newFilter(); + try { + newFilter(); + } catch (Exception e) { + // Ignorable, testnet starts empty, log for debugging + log.warn(e.getMessage()); + } return; } try { - TreeMap filters = new TreeMap<>(); // Load all data from all filters - List bytes = persistenceProvider.loadAllKeysFromTable(CuckooBucket.class); - for (byte[] bucketData : bytes) { - CuckooBucket bucket = new CuckooBucket(); - bucket.read(bucketData); + List bytes = persistenceProvider.loadAllKeysFromTable(Cuckoo.class); + for (byte[] filterData : bytes) { + Cuckoo bucket = new Cuckoo(); + bucket.read(filterData); - if (MAX_FILTERS < bucket.bucketId.getValue()) { + if (MAX_FILTERS < bucket.filterId.getValue()) { throw new PrunedTransactionException("Database contains more filters then we can store"); } - // Find its bucket, or create it if wasnt there - CuckooFilter filter = filters.get(bucket.bucketId.getValue()); - if (null == filter) { - filter = new CuckooFilterImpl(filterSize, 4, 16); - filters.put(bucket.bucketId.getValue(), filter); - } - - // Don't update using the entire CuckooBucket so that packages are separate-able - filter.update(bucket.bucketIndex.getValue(), bucket.bucketBits); + CuckooFilter filter = new CuckooFilterImpl(filterSize, 4, 16, bucket.filterBits); + filters.put(bucket.filterId.getValue(), filter); } //Then add all in order, treemap maintains order from lowest to highest key @@ -138,15 +135,23 @@ private void readPreviousPrunedTransactions() throws PrunedTransactionException newFilter(); } - } catch (IllegalArgumentException e) { + } catch (Exception e) { throw new PrunedTransactionException(e); } } - private void persistFilter(CuckooFilter filter, Integer index) { - + private void persistFilter(CuckooFilter filter, Integer index) throws Exception { + IntegerIndex intIndex = new IntegerIndex(index); + Cuckoo bucket = new Cuckoo(); + bucket.filterId = intIndex; + bucket.filterBits = filter.getFilterData(); + persistenceProvider.save(bucket, intIndex); } + /** + * + * {@inheritDoc} + */ @Override public boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException { byte[] hashBytes = transactionHash.bytes(); @@ -162,17 +167,28 @@ public boolean containsTransaction(Hash transactionHash) throws PrunedTransactio } /** + * Adds a transaction to the latest filter. When this filter is full, saves and creates a new filter. * * {@inheritDoc} + * @throws PrunedTransactionException when saving the old (full) filter or deleting the oldest fails */ @Override public void addTransaction(Hash transactionHash) throws PrunedTransactionException { - if (null == lastAddedFilter) { - newFilter(); - } - - if (!lastAddedFilter.add(transactionHash.bytes())){ - newFilter().add(transactionHash.bytes()); + try { + if (null == lastAddedFilter) { + newFilter(); + } + + if (!lastAddedFilter.add(transactionHash.bytes())){ + try { + persistFilter(lastAddedFilter, highestIndex); + } catch (Exception e) { + throw new PrunedTransactionException(e); + } + newFilter().add(transactionHash.bytes()); + } + } catch (Exception e) { + throw new PrunedTransactionException(e); } } @@ -187,12 +203,17 @@ public void addTransactionBatch(Collection transactionHashes) throws Prune for (Hash transactionHash : transactionHashes) { addTransaction(transactionHash); } - persistFilter(lastAddedFilter, highestIndex); + try { + persistFilter(lastAddedFilter, highestIndex); + } catch (Exception e) { + throw new PrunedTransactionException(e); + } } - private CuckooFilter newFilter() { + private CuckooFilter newFilter() throws Exception { if (filters.isAtFullCapacity()) { log.debug("Removing " + filters.peek()); + persistenceProvider.delete(Cuckoo.class, new IntegerIndex(getLowestIndex())); } highestIndex++; @@ -201,4 +222,12 @@ private CuckooFilter newFilter() { filters.offer(lastAddedFilter = new CuckooFilterImpl(filterSize, 4, 16)); return lastAddedFilter; } + + private int getLowestIndex() { + if (highestIndex < MAX_FILTERS) { + return 0; + } + + return highestIndex - MAX_FILTERS; + } } diff --git a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java index 8cabd828ef..4c69ee31c3 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java +++ b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java @@ -2,7 +2,7 @@ import java.util.BitSet; -import com.iota.iri.model.persistables.CuckooBucket; +import com.iota.iri.model.persistables.Cuckoo; /** * The Cuckoo Filter is a probabilistic data structure that supports fast set membership testing. @@ -89,12 +89,8 @@ public interface CuckooFilter { int size(); /** - * Update a part of this filter with the bucket information supplied. - * Discards any previous data inside this bucket. - * - * @param index The index of the bucket we are updating - * @param bits the bucket data we use to update the filter - * @throws IllegalArgumentException when the bucket does not contain data which is valid for this filter + * This method returns a copy of all the bits that make up this filter + * @return The ilter bits */ - void update(int index, BitSet bits) throws IllegalArgumentException ; + BitSet getFilterData(); } diff --git a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java index 6d9f75b601..e7ff51c68c 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java +++ b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java @@ -1,6 +1,5 @@ package com.iota.iri.utils.datastructure.impl; -import com.iota.iri.model.persistables.CuckooBucket; import com.iota.iri.utils.BitSetUtils; import com.iota.iri.utils.datastructure.CuckooFilter; @@ -123,6 +122,38 @@ public CuckooFilterImpl(int itemCount, int bucketSize, int fingerPrintSize) thro cuckooFilterTable = new CuckooFilterTable(tableSize, bucketSize, fingerPrintSize); } + + /** + * Advanced constructor that allows for fine tuning of the desired filter. + * + * It first saves a reference to the hash function and then checks the parameters - the finger print size cannot + * be bigger than 128 bits because SHA1 generates 160 bits and we use 128 of that for the fingerprint and the rest + * for the index. + * + * After verifying that the passed in parameters are reasonable, we calculate the required size of the + * {@link CuckooFilterTable} by increasing the table size exponentially until we can fit the desired item count with + * a load factor of <= 0.955. Finally we create the {@link CuckooFilterTable} that will hold our data. + * + * NOTE: The actual size will be slightly bigger since the size has to be a power of 2 and take the optimal load + * factor of 0.955 into account. + * + * @param itemCount the minimum amount of items that should fit into the filter + * @param bucketSize the amount of items that can be stored in each bucket + * @param fingerPrintSize the amount of bits per fingerprint (it has to be bigger than 0 and smaller than 128) + * @param filterData The data this filter is initialized with. Must match with the parameters + * @throws IllegalArgumentException if the finger print size is too small or too big + * @throws InternalError if the SHA1 hashing function can not be found with this java version [should never happen] + */ + public CuckooFilterImpl(int itemCount, int bucketSize, int fingerPrintSize, BitSet filterData) throws IllegalArgumentException, + InternalError { + + this(itemCount, bucketSize, fingerPrintSize); + + if (cuckooFilterTable.data.size() != filterData.size()) { + throw new IllegalArgumentException("Filter data does not match filter parameters"); + } + cuckooFilterTable.data = BitSet.valueOf(filterData.toByteArray()); + } /** * {@inheritDoc} @@ -143,31 +174,6 @@ public boolean add(String item) throws IndexOutOfBoundsException { public boolean add(byte[] item) throws IndexOutOfBoundsException { return add(new CuckooFilterItem(hashFunction.digest(item))); } - /** - * {@inheritDoc} - * - */ - @Override - public void update(int index, BitSet bits) throws IllegalArgumentException { - int amountInBucket = bucketSize; - if (bits.length() % fingerPrintSize != 0) { - throw new IllegalArgumentException("Provided bits do not match fingerprint scheme"); - } else if (bits.length() % fingerPrintSize > amountInBucket) { - throw new IllegalArgumentException("Provided fingerprint data will overflow the bucket"); - } else if (index > tableSize * 0.955) { - throw new IllegalArgumentException("Provided bucket exceeds filter size"); - } else if (false) { - // Can we recover input in any case when expected input does not match given? - } - - for (int i=0; i < amountInBucket; i++) { - cuckooFilterTable.set(index, i, bits.get( - fingerPrintSize * i, - fingerPrintSize * (i + 1) - )); - - } - } /** * {@inheritDoc} @@ -465,6 +471,16 @@ private BitSet generateFingerPrint(byte[] hash) throws IllegalArgumentException // do a simple conversion of the byte array to a BitSet of the desired length return BitSetUtils.convertByteArrayToBitSet(hash, 4, fingerPrintSize); } + + @Override + public String toString() { + return index + ""; + } + + @Override + public BitSet getFilterData() { + return (BitSet) cuckooFilterTable.data.clone(); + } /** * Internal helper class to represent items that are stored in the filter. @@ -617,9 +633,4 @@ public CuckooFilterTable delete(int bucketIndex, int slotIndex) { return set(bucketIndex, slotIndex, null); } } - - @Override - public String toString() { - return index +" "; - } } From 8c2ade28cb7ca9457af733e73250917c0380d3a5 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Sun, 14 Apr 2019 20:47:56 +0200 Subject: [PATCH 04/11] Added tests --- .../impl/PrunedTransactionProviderImpl.java | 40 +++++++++----- .../PrunedTransactionProviderImplTest.java | 54 ++++++++++++++----- 2 files changed, 69 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java index aa3087d357..9493dee683 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -2,11 +2,8 @@ import java.util.Collection; import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Map.Entry; -import java.util.Set; import java.util.TreeMap; import org.apache.commons.collections4.queue.CircularFifoQueue; @@ -20,7 +17,6 @@ import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.transactionpruning.PrunedTransactionException; import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; -import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.datastructure.CuckooFilter; @@ -33,6 +29,10 @@ public class PrunedTransactionProviderImpl implements PrunedTransactionProvider private static final Logger log = LoggerFactory.getLogger(PrunedTransactionProviderImpl.class); + private static final int BUCKET_SIZE = 4; + + private static final int FINGER_PRINT_SIZE = 16; + private static final int FILTER_SIZE = 200000; private static final int MAX_FILTERS = 10; @@ -44,21 +44,23 @@ public class PrunedTransactionProviderImpl implements PrunedTransactionProvider private CircularFifoQueue filters; private CuckooFilter lastAddedFilter; - // Once this exceeds integer max range, this will give problems. + // Once this exceeds integer max range, this will give problems. + // Requires max int * FILTER_SIZE * BUCKET_SIZE * ~1.005 transactions (138 billion) + // or max int * 5 min (milestone avg. time) minutes (20428 years) private Integer highestIndex = -1; private int filterSize; /** - * + * Creates a transaction provider with the default filter size {@value #FILTER_SIZE} */ public PrunedTransactionProviderImpl() { this(FILTER_SIZE); } /** - * - * @param filterSize + * Creates a transaction provider with a custom filter size + * @param filterSize the size each cuckoo filter will have (before resizing) */ public PrunedTransactionProviderImpl(int filterSize) { this.filterSize = filterSize; @@ -116,7 +118,7 @@ private void readPreviousPrunedTransactions() throws PrunedTransactionException throw new PrunedTransactionException("Database contains more filters then we can store"); } - CuckooFilter filter = new CuckooFilterImpl(filterSize, 4, 16, bucket.filterBits); + CuckooFilter filter = new CuckooFilterImpl(filterSize, BUCKET_SIZE, FINGER_PRINT_SIZE, bucket.filterBits); filters.put(bucket.filterId.getValue(), filter); } @@ -149,16 +151,22 @@ private void persistFilter(CuckooFilter filter, Integer index) throws Exception } /** + * Checks if our filters contain this transaction hash, starting with the most recent filter * * {@inheritDoc} */ @Override public boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException { byte[] hashBytes = transactionHash.bytes(); - + // last filter added is most recent, thus most likely to be requested + if (lastAddedFilter.contains(hashBytes)) { + return true; + } + + // Loop over other filters in order of recently added, skip first CuckooFilter[] filterArray = filters.toArray(new CuckooFilter[filters.size()]); - for (int i = filterArray.length - 1; i >=0; i--) { + for (int i = filterArray.length - 2; i >=0; i--) { if (filterArray[i].contains(hashBytes)) { return true; } @@ -179,7 +187,7 @@ public void addTransaction(Hash transactionHash) throws PrunedTransactionExcepti newFilter(); } - if (!lastAddedFilter.add(transactionHash.bytes())){ + if (!lastAddedFilter.add(transactionHash.bytes()) || shouldSwitchFilter()){ try { persistFilter(lastAddedFilter, highestIndex); } catch (Exception e) { @@ -200,6 +208,7 @@ public void addTransaction(Hash transactionHash) throws PrunedTransactionExcepti public void addTransactionBatch(Collection transactionHashes) throws PrunedTransactionException { // Should we create a new filter when we get this method call? // It probably implies a new pruned job completed, and these TX would be checked more often then the older ones. + // However, we have less filters to use/faster filter deletion for (Hash transactionHash : transactionHashes) { addTransaction(transactionHash); } @@ -219,10 +228,15 @@ private CuckooFilter newFilter() throws Exception { highestIndex++; // We keep a reference to the last filter to prevent looking it up every time - filters.offer(lastAddedFilter = new CuckooFilterImpl(filterSize, 4, 16)); + filters.offer(lastAddedFilter = new CuckooFilterImpl(filterSize, BUCKET_SIZE, FINGER_PRINT_SIZE)); return lastAddedFilter; } + private boolean shouldSwitchFilter() { + // Cuckoo filter speed/accuracy is best when filters stay partially empty + return lastAddedFilter != null && lastAddedFilter.size() > filterSize; + } + private int getLowestIndex() { if (highestIndex < MAX_FILTERS) { return 0; diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java index e609c4654d..99e9a0470d 100644 --- a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java @@ -38,8 +38,7 @@ public void setUp() throws PrunedTransactionException { Mockito.when(config.getPrunedTransactionsDbPath()).thenReturn(dbFolder.getRoot().getAbsolutePath()); Mockito.when(config.getPrunedTransactionsDbLogPath()).thenReturn(logFolder.getRoot().getAbsolutePath()); - // Filter size of 10, for easier testing - provider = new PrunedTransactionProviderImpl(10); + provider = new PrunedTransactionProviderImpl(10000); provider.init(config); } @@ -49,21 +48,52 @@ public void tearDown() { } @Test - public void test() throws PrunedTransactionException { - Hash randomHash = TransactionTestUtils.getRandomTransactionHash(); - provider.addTransaction(randomHash); - for (int i=0; i<100; i++) { - provider.addTransaction(TransactionTestUtils.getRandomTransactionHash()); + public void containsMarginalOkayTest() throws PrunedTransactionException { + int size = 10000; + int contains = 0; + + Hash[] hashes = new Hash[size]; + for (int i=0; i 90); + // 0.05% margin of spents for unknowns + assertTrue("Provider should only contain the added hash", contains < size*1.0005); + } + + @Test + public void deletedFiltersTest() throws PrunedTransactionException { + int size = 100; + Hash[] hashes = new Hash[size]; + for (int i=0; i size*0.995); } } From 54fd525cf0e57a3a2858b69ff614a3aebc285771 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Mon, 15 Apr 2019 20:33:11 +0200 Subject: [PATCH 05/11] Added extra javadoc --- .../impl/PrunedTransactionProviderImpl.java | 15 ++++++++++++++- .../PrunedTransactionProviderImplTest.java | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java index 9493dee683..a7c1ab7f53 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -23,18 +23,31 @@ import com.iota.iri.utils.datastructure.impl.CuckooFilterImpl; /** - * + * Implementation of a pruned transaction provider which uses a cuckoo filter to store pruned hashes */ public class PrunedTransactionProviderImpl implements PrunedTransactionProvider { private static final Logger log = LoggerFactory.getLogger(PrunedTransactionProviderImpl.class); + /** + * The amount of fingerprints each bucket keeps. + * More than 4 + */ private static final int BUCKET_SIZE = 4; + /** + * The amount of bits a hash gets transformed to (Higher is better, but uses more storage) + */ private static final int FINGER_PRINT_SIZE = 16; + /** + * The estimated amount of fingerprints each filter should hold (Will be scaled by cuckoo impl.) + */ private static final int FILTER_SIZE = 200000; + /** + * The maximum amount of filters we have in use at any given time + */ private static final int MAX_FILTERS = 10; private RocksDBPersistenceProvider persistenceProvider; diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java index 99e9a0470d..6ba62a86f3 100644 --- a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java @@ -86,8 +86,8 @@ public void deletedFiltersTest() throws PrunedTransactionException { for (int i=0; i<10000*10; i++) { provider.addTransaction(TransactionTestUtils.getRandomTransactionHash()); } - + // by now, we have added 10 full filters, so the first 100 should not exist anymore (deleted) int notContains = 0; for (int i=0; i Date: Wed, 24 Apr 2019 15:54:21 +0200 Subject: [PATCH 06/11] Added javadoc and cleaned import --- src/main/java/com/iota/iri/model/persistables/Cuckoo.java | 2 +- .../java/com/iota/iri/utils/datastructure/CuckooFilter.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/com/iota/iri/model/persistables/Cuckoo.java b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java index 780f18bfa4..6e397c759a 100644 --- a/src/main/java/com/iota/iri/model/persistables/Cuckoo.java +++ b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java @@ -19,7 +19,7 @@ public class Cuckoo implements Persistable { public IntegerIndex filterId; /** - * + * The bits that make up the CF */ public BitSet filterBits; diff --git a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java index 4c69ee31c3..e60c6aad0e 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java +++ b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java @@ -2,8 +2,6 @@ import java.util.BitSet; -import com.iota.iri.model.persistables.Cuckoo; - /** * The Cuckoo Filter is a probabilistic data structure that supports fast set membership testing. * From d1b1d2a6cf05420006a355f3e874738941c92d6b Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 24 Apr 2019 15:59:06 +0200 Subject: [PATCH 07/11] Fixed naming error in config --- src/main/java/com/iota/iri/conf/BaseIotaConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index cd7efdf078..7fb07657de 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -654,7 +654,7 @@ public String getSpentAddressesDbLogPath() { @JsonProperty @Parameter(names = {"--spent-addresses-db-log-path"}, description = SnapshotConfig.Descriptions.SPENT_ADDRESSES_DB_LOG_PATH) - protected void setPrunedTransactionDbLogPath(String spentAddressesDbLogPath) { + protected void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) { this.spentAddressesDbLogPath = spentAddressesDbLogPath; } @@ -665,7 +665,7 @@ public String getPrunedTransactionsDbLogPath() { @JsonProperty @Parameter(names = {"--pruned-transactions-db-log-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_LOG_PATH) - protected void setSpentAddressesDbLogPath(String prunedTransactionsDbLogPath) { + protected void setPrunedTransactionDbLogPath(String prunedTransactionsDbLogPath) { this.prunedTransactionsDbLogPath = prunedTransactionsDbLogPath; } From 998d8ec2c3d0c06b0b671c9d8e9a70ffa7a227b2 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 24 Apr 2019 16:04:13 +0200 Subject: [PATCH 08/11] Removed/updated more comments, deleted unused file --- .../async/AsyncTransactionPruner.java | 2 +- .../impl/PrunedTransactionProviderImpl.java | 2 +- .../jobs/HashPruningJob.java | 19 ------------------- .../jobs/MilestonePrunerJob.java | 4 ---- 4 files changed, 2 insertions(+), 25 deletions(-) delete mode 100644 src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index 99a47edc62..6b4a9c3d95 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -106,7 +106,7 @@ public class AsyncTransactionPruner implements TransactionPruner { private final Map, JobQueue> jobQueues = new HashMap<>(); /** - * + * Provider for managing transactions we delete from the database in an optimized data structure */ private PrunedTransactionProvider prunedTransactionProvider; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java index a7c1ab7f53..afc197acd9 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -31,7 +31,7 @@ public class PrunedTransactionProviderImpl implements PrunedTransactionProvider /** * The amount of fingerprints each bucket keeps. - * More than 4 + * More than 4 will be slower in performance, whilst the extra space it gets us is not needed. */ private static final int BUCKET_SIZE = 4; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java deleted file mode 100644 index 7a0d6a4fc8..0000000000 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/HashPruningJob.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.iota.iri.service.transactionpruning.jobs; - -import com.iota.iri.service.transactionpruning.TransactionPruningException; - -public class HashPruningJob extends AbstractTransactionPrunerJob { - - public HashPruningJob() { - - } - - @Override - public void process() throws TransactionPruningException { - } - - @Override - public String serialize() { - return null; - } -} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index 62d25fe816..a6eff17628 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -234,10 +234,6 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { } catch (TransactionPruningException e) { throw new RuntimeException(e); } - } else { - // Confirmed TX, add to data structure - // either add to list, or add a new job to add to the cuckoofilter - } } else if(Milestone.class.equals(element.hi)) { MilestoneViewModel.clear(((IntegerIndex) element.low).getValue()); From eafce427ccae08aefb2cfc38f3c5bf14d840a48e Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 24 Apr 2019 20:43:58 +0200 Subject: [PATCH 09/11] Documented serializer --- .../java/com/iota/iri/utils/Serializer.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/main/java/com/iota/iri/utils/Serializer.java b/src/main/java/com/iota/iri/utils/Serializer.java index f46e43edb5..967e809468 100644 --- a/src/main/java/com/iota/iri/utils/Serializer.java +++ b/src/main/java/com/iota/iri/utils/Serializer.java @@ -37,13 +37,36 @@ public static long getLong(byte[] bytes, int start) { return res; } + /** + * Reads the default amount of bytes for an int(4) and turns it into an int. + * Starts at the beginning of the array + * + * @param bytes The bytes we use to make an int + * @return The created int, or 0 when bytes is null + */ public static int getInteger(byte[] bytes) { return getInteger(bytes, 0); } + + /** + * Reads the default amount of bytes for an int(4) and turns it into an int. + * + * @param bytes The bytes we use to make an int + * @param start The point in the array from which we start reading + * @return The created int, or 0 when bytes is null + */ public static int getInteger(byte[] bytes, int start) { return getInteger(bytes, start, Integer.BYTES); } + /** + * Reads the bytes for the given length, starting at the starting point given. + * + * @param bytes The bytes we use to make an int + * @param start The point in the array from which we start reading + * @param length Amount of bytes to read + * @return The created int, or 0 when bytes is null + */ public static int getInteger(byte[] bytes, int start, int length) { if(bytes == null) { return 0; From 9f64545dd0b23bd5758631f0275f86656cda9797 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 24 Apr 2019 20:44:13 +0200 Subject: [PATCH 10/11] Removed unused index in CF --- .../iri/utils/datastructure/impl/CuckooFilterImpl.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java index e7ff51c68c..83bd08840c 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java +++ b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java @@ -12,10 +12,6 @@ */ public class CuckooFilterImpl implements CuckooFilter { - private static int CUR_INDEX = 0; - - int index = CUR_INDEX ++; - /** * The amount of times we try to kick elements when inserting before we consider the index to be too full. */ @@ -471,11 +467,6 @@ private BitSet generateFingerPrint(byte[] hash) throws IllegalArgumentException // do a simple conversion of the byte array to a BitSet of the desired length return BitSetUtils.convertByteArrayToBitSet(hash, 4, fingerPrintSize); } - - @Override - public String toString() { - return index + ""; - } @Override public BitSet getFilterData() { From a35f8410adb0a38d939484ddac703146c4c9cde4 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Tue, 14 May 2019 18:19:15 +0200 Subject: [PATCH 11/11] Fixed test method usage --- .../PrunedTransactionProviderImplTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java index 6ba62a86f3..803e56ef91 100644 --- a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java @@ -54,7 +54,7 @@ public void containsMarginalOkayTest() throws PrunedTransactionException { Hash[] hashes = new Hash[size]; for (int i=0; i