diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index 1c2e2288e1..c203e547e1 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -5,7 +5,6 @@ import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.iota.iri.IRI; import com.iota.iri.crypto.SpongeFactory; import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; @@ -111,7 +110,9 @@ public abstract class BaseIotaConfig implements IotaConfig { protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH; protected String spentAddressesDbPath = Defaults.SPENT_ADDRESSES_DB_PATH; protected String spentAddressesDbLogPath = Defaults.SPENT_ADDRESSES_DB_LOG_PATH; - + protected String prunedTransactionsDbPath = Defaults.PRUNED_TRANSACTIONS_DB_PATH; + protected String prunedTransactionsDbLogPath = Defaults.PRUNED_TRANSACTIONS_DB_LOG_PATH; + public BaseIotaConfig() { //empty constructor } @@ -667,13 +668,35 @@ protected void setSpentAddressesDbPath(String spentAddressesDbPath) { public String getSpentAddressesDbLogPath() { return spentAddressesDbLogPath; } - + @JsonProperty @Parameter(names = {"--spent-addresses-db-log-path"}, description = SnapshotConfig.Descriptions.SPENT_ADDRESSES_DB_LOG_PATH) protected void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) { this.spentAddressesDbLogPath = spentAddressesDbLogPath; } - + + @Override + public String getPrunedTransactionsDbLogPath() { + return prunedTransactionsDbLogPath; + } + + @JsonProperty + @Parameter(names = {"--pruned-transactions-db-log-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_LOG_PATH) + protected void setPrunedTransactionDbLogPath(String prunedTransactionsDbLogPath) { + this.prunedTransactionsDbLogPath = prunedTransactionsDbLogPath; + } + + @Override + public String getPrunedTransactionsDbPath() { + return prunedTransactionsDbPath; + } + + @JsonProperty + @Parameter(names = {"--pruned-transactions-db-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_PATH) + protected void setPrunedTransactionsDbPath(String prunedTransactionsDbPath) { + this.prunedTransactionsDbPath = prunedTransactionsDbPath; + } + /** * Checks if ZMQ is enabled. * @return true if zmqEnableTcp or zmqEnableIpc is set. @@ -951,6 +974,9 @@ public interface Defaults { int LOCAL_SNAPSHOTS_DEPTH_MIN = 100; String SPENT_ADDRESSES_DB_PATH = "spent-addresses-db"; String SPENT_ADDRESSES_DB_LOG_PATH = "spent-addresses-log"; + + String PRUNED_TRANSACTIONS_DB_LOG_PATH = "spent-addresses-db"; + String PRUNED_TRANSACTIONS_DB_PATH = "spent-addresses-log"; String LOCAL_SNAPSHOTS_BASE_PATH = "mainnet"; String SNAPSHOT_FILE = "/snapshotMainnet.txt"; diff --git a/src/main/java/com/iota/iri/conf/SnapshotConfig.java b/src/main/java/com/iota/iri/conf/SnapshotConfig.java index 33c30fb705..b213c4f8b6 100644 --- a/src/main/java/com/iota/iri/conf/SnapshotConfig.java +++ b/src/main/java/com/iota/iri/conf/SnapshotConfig.java @@ -76,6 +76,16 @@ public interface SnapshotConfig extends Config { */ String getSpentAddressesDbLogPath(); + /** + * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_PATH} + */ + String getPrunedTransactionsDbPath(); + + /** + * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_LOG_PATH} + */ + String getPrunedTransactionsDbLogPath(); + interface Descriptions { String LOCAL_SNAPSHOTS_ENABLED = "Flag that determines if local snapshots are enabled."; @@ -94,5 +104,8 @@ interface Descriptions { "from previous epochs"; String SPENT_ADDRESSES_DB_PATH = "The folder where the spent addresses DB saves its data."; String SPENT_ADDRESSES_DB_LOG_PATH = "The folder where the spent addresses DB saves its logs."; + String PRUNED_TRANSACTIONS_DB_PATH = "The folder where the pruned transactions DB saves its data."; + String PRUNED_TRANSACTIONS_DB_LOG_PATH = "The folder where the pruned transactions DB saves its logs."; } + } diff --git a/src/main/java/com/iota/iri/model/persistables/Cuckoo.java b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java new file mode 100644 index 0000000000..6e397c759a --- /dev/null +++ b/src/main/java/com/iota/iri/model/persistables/Cuckoo.java @@ -0,0 +1,82 @@ +package com.iota.iri.model.persistables; + +import java.util.BitSet; + +import org.apache.commons.lang3.ArrayUtils; + +import com.iota.iri.model.IntegerIndex; +import com.iota.iri.storage.Persistable; +import com.iota.iri.utils.Serializer; + +/** + * Persistable to manage the data we get as a result of pruning a milestone and its transactions + */ +public class Cuckoo implements Persistable { + + /** + * The filter number it belonged to in previous cycles + */ + public IntegerIndex filterId; + + /** + * The bits that make up the CF + */ + public BitSet filterBits; + + /** + * + * {@inheritDoc} + */ + @Override + public byte[] bytes() { + byte[] num = filterId.bytes(); + return ArrayUtils.addAll(num, filterBits.toByteArray()); + } + + /** + * Reads a CuckooBucket from the provided bytes. + * First 4 bytes are the bucket id, second 4 are the index inside that bucket + * Rest of the bytes is the bucket data + * + * {@inheritDoc} + */ + @Override + public void read(byte[] bytes) { + if(bytes != null) { + filterId = new IntegerIndex(Serializer.getInteger(bytes, 0)); + + short start = 4; + filterBits = new BitSet(bytes.length - start); + for (int i = start; i < bytes.length; i++) { + filterBits.set(i-start, bytes[i]); + } + } + } + + /** + * + * {@inheritDoc} + */ + @Override + public byte[] metadata() { + return new byte[0]; + } + + /** + * + * {@inheritDoc} + */ + @Override + public void readMetadata(byte[] bytes) { + } + + /** + * + * {@inheritDoc} + */ + @Override + public boolean merge() { + return false; + } + +} diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index c59c61dc6e..8bcc7503c7 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -54,8 +54,7 @@ public SpentAddressesProviderImpl init(SnapshotConfig config) {{put("spent-addresses", SpentAddress.class);}}, null); this.rocksDBPersistenceProvider.init(); readPreviousEpochsSpentAddresses(); - } - catch (Exception e) { + } catch (Exception e) { throw new SpentAddressesException("There is a problem with accessing stored spent addresses", e); } return this; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java new file mode 100644 index 0000000000..4e1ea756da --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java @@ -0,0 +1,38 @@ +package com.iota.iri.service.transactionpruning; + +/** + * This class is used to wrap exceptions that are specific to the pruned transaction persistance. + * + * It allows us to distinct between the different kinds of errors that can happen during the execution of the code. + */ +public class PrunedTransactionException extends Exception { + /** + * Constructor of the exception which allows us to provide a specific error message and the cause of the error. + * + * @param message reason why this error occurred + * @param cause wrapped exception that caused this error + */ + public PrunedTransactionException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor of the exception which allows us to provide a specific error message without having an underlying + * cause. + * + * @param message reason why this error occurred + */ + public PrunedTransactionException(String message) { + super(message); + } + + /** + * Constructor of the exception which allows us to wrap the underlying cause of the error without providing a + * specific reason. + * + * @param cause wrapped exception that caused this error + */ + public PrunedTransactionException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java new file mode 100644 index 0000000000..ea449b2f22 --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java @@ -0,0 +1,36 @@ +package com.iota.iri.service.transactionpruning; + +import java.util.Collection; + +import com.iota.iri.model.Hash; + +/** + * Find, mark and store pruned transactions + */ +public interface PrunedTransactionProvider { + + /** + * Checks if this transactions has been pruned + * + * @param transactionHash The transaction to check for + * @return true if it is, else false + * @throws PrunedTransactionException If the provider fails to check the transaction + */ + boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException; + + /** + * Mark a transaction as spent. + * + * @param transactionHash the transaction which we want to mark. + * @throws PrunedTransactionException If the provider fails to add the transaction + */ + void addTransaction(Hash transactionHash) throws PrunedTransactionException; + + /** + * Mark all transactions as pruned. + * + * @param transactionHashes The transactions we want to mark + * @throws PrunedTransactionException If the provider fails to add a transaction + */ + void addTransactionBatch(Collection transactionHashes) throws PrunedTransactionException; +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java index 619b59a5e9..5923cd5bab 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java @@ -121,6 +121,20 @@ public interface TransactionPrunerJob { * @param transactionPrunerJobStatus new execution status of the job */ void setStatus(TransactionPrunerJobStatus transactionPrunerJobStatus); + + /** + * Getter for the pruned transaction maintainer. + * + * @return pruned transaction maintainer of the job. + */ + PrunedTransactionProvider getPrunedProvider(); + + /** + * Setter for the pruned transaction maintainer. + * + * @param prunedTransactionProvider pruned transaction maintainer of the job + */ + void setPrunedProvider(PrunedTransactionProvider prunedTransactionProvider); /** * This method processes the cleanup job and performs the actual pruning. diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index 4d9e0632a3..ed35252223 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -5,6 +5,7 @@ import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPruningException; @@ -110,6 +111,11 @@ public class AsyncTransactionPruner implements TransactionPruner { */ private final Map, JobQueue> jobQueues = new HashMap<>(); + /** + * Provider for managing transactions we delete from the database in an optimized data structure + */ + private PrunedTransactionProvider prunedTransactionProvider; + /** * This method initializes the instance and registers its dependencies.
*
@@ -158,6 +164,7 @@ public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvi @Override public void addJob(TransactionPrunerJob job) throws TransactionPruningException { job.setTransactionPruner(this); + job.setPrunedProvider(prunedTransactionProvider); job.setSpentAddressesService(spentAddressesService); job.setSpentAddressesProvider(spentAddressesProvider); job.setTangle(tangle); diff --git a/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java new file mode 100644 index 0000000000..afc197acd9 --- /dev/null +++ b/src/main/java/com/iota/iri/service/transactionpruning/impl/PrunedTransactionProviderImpl.java @@ -0,0 +1,260 @@ +package com.iota.iri.service.transactionpruning.impl; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.commons.collections4.queue.CircularFifoQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.model.Hash; +import com.iota.iri.model.IntegerIndex; +import com.iota.iri.model.persistables.Cuckoo; +import com.iota.iri.service.spentaddresses.SpentAddressesException; +import com.iota.iri.service.transactionpruning.PrunedTransactionException; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; +import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; +import com.iota.iri.utils.datastructure.CuckooFilter; +import com.iota.iri.utils.datastructure.impl.CuckooFilterImpl; + +/** + * Implementation of a pruned transaction provider which uses a cuckoo filter to store pruned hashes + */ +public class PrunedTransactionProviderImpl implements PrunedTransactionProvider { + + private static final Logger log = LoggerFactory.getLogger(PrunedTransactionProviderImpl.class); + + /** + * The amount of fingerprints each bucket keeps. + * More than 4 will be slower in performance, whilst the extra space it gets us is not needed. + */ + private static final int BUCKET_SIZE = 4; + + /** + * The amount of bits a hash gets transformed to (Higher is better, but uses more storage) + */ + private static final int FINGER_PRINT_SIZE = 16; + + /** + * The estimated amount of fingerprints each filter should hold (Will be scaled by cuckoo impl.) + */ + private static final int FILTER_SIZE = 200000; + + /** + * The maximum amount of filters we have in use at any given time + */ + private static final int MAX_FILTERS = 10; + + private RocksDBPersistenceProvider persistenceProvider; + + private SnapshotConfig config; + + private CircularFifoQueue filters; + private CuckooFilter lastAddedFilter; + + // Once this exceeds integer max range, this will give problems. + // Requires max int * FILTER_SIZE * BUCKET_SIZE * ~1.005 transactions (138 billion) + // or max int * 5 min (milestone avg. time) minutes (20428 years) + private Integer highestIndex = -1; + + private int filterSize; + + /** + * Creates a transaction provider with the default filter size {@value #FILTER_SIZE} + */ + public PrunedTransactionProviderImpl() { + this(FILTER_SIZE); + } + + /** + * Creates a transaction provider with a custom filter size + * @param filterSize the size each cuckoo filter will have (before resizing) + */ + public PrunedTransactionProviderImpl(int filterSize) { + this.filterSize = filterSize; + } + + /** + * Starts the PrunedTransactionProvider by reading the current pruned transactions from a persistence provider + * + * @param config The snapshot configuration used for file location + * @return the current instance + * @throws SpentAddressesException if we failed to create a file at the designated location + */ + public PrunedTransactionProviderImpl init(SnapshotConfig config) throws PrunedTransactionException { + this.config = config; + try { + this.persistenceProvider = new RocksDBPersistenceProvider( + config.getPrunedTransactionsDbPath(), + config.getPrunedTransactionsDbLogPath(), + 1000, + new HashMap>(1) + {{put("pruned-transactions", Cuckoo.class);}}, null); + this.persistenceProvider.init(); + + filters = new CircularFifoQueue(MAX_FILTERS); + + readPreviousPrunedTransactions(); + } catch (Exception e) { + throw new PrunedTransactionException("There is a problem with accessing previously pruned transactions", e); + } + + return this; + } + + private void readPreviousPrunedTransactions() throws PrunedTransactionException { + if (config.isTestnet()) { + try { + newFilter(); + } catch (Exception e) { + // Ignorable, testnet starts empty, log for debugging + log.warn(e.getMessage()); + } + return; + } + + try { + TreeMap filters = new TreeMap<>(); + + // Load all data from all filters + List bytes = persistenceProvider.loadAllKeysFromTable(Cuckoo.class); + for (byte[] filterData : bytes) { + Cuckoo bucket = new Cuckoo(); + bucket.read(filterData); + + if (MAX_FILTERS < bucket.filterId.getValue()) { + throw new PrunedTransactionException("Database contains more filters then we can store"); + } + + CuckooFilter filter = new CuckooFilterImpl(filterSize, BUCKET_SIZE, FINGER_PRINT_SIZE, bucket.filterBits); + filters.put(bucket.filterId.getValue(), filter); + } + + //Then add all in order, treemap maintains order from lowest to highest key + for (CuckooFilter filter : filters.values()) { + if (null != filter) { + this.filters.add(filter); + } + } + + Entry entry = filters.lastEntry(); + if (null != entry) { + lastAddedFilter = entry.getValue(); + highestIndex = entry.getKey(); + } else { + newFilter(); + } + + } catch (Exception e) { + throw new PrunedTransactionException(e); + } + } + + private void persistFilter(CuckooFilter filter, Integer index) throws Exception { + IntegerIndex intIndex = new IntegerIndex(index); + Cuckoo bucket = new Cuckoo(); + bucket.filterId = intIndex; + bucket.filterBits = filter.getFilterData(); + persistenceProvider.save(bucket, intIndex); + } + + /** + * Checks if our filters contain this transaction hash, starting with the most recent filter + * + * {@inheritDoc} + */ + @Override + public boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException { + byte[] hashBytes = transactionHash.bytes(); + + // last filter added is most recent, thus most likely to be requested + if (lastAddedFilter.contains(hashBytes)) { + return true; + } + + // Loop over other filters in order of recently added, skip first + CuckooFilter[] filterArray = filters.toArray(new CuckooFilter[filters.size()]); + for (int i = filterArray.length - 2; i >=0; i--) { + if (filterArray[i].contains(hashBytes)) { + return true; + } + } + return false; + } + + /** + * Adds a transaction to the latest filter. When this filter is full, saves and creates a new filter. + * + * {@inheritDoc} + * @throws PrunedTransactionException when saving the old (full) filter or deleting the oldest fails + */ + @Override + public void addTransaction(Hash transactionHash) throws PrunedTransactionException { + try { + if (null == lastAddedFilter) { + newFilter(); + } + + if (!lastAddedFilter.add(transactionHash.bytes()) || shouldSwitchFilter()){ + try { + persistFilter(lastAddedFilter, highestIndex); + } catch (Exception e) { + throw new PrunedTransactionException(e); + } + newFilter().add(transactionHash.bytes()); + } + } catch (Exception e) { + throw new PrunedTransactionException(e); + } + } + + /** + * + * {@inheritDoc} + */ + @Override + public void addTransactionBatch(Collection transactionHashes) throws PrunedTransactionException { + // Should we create a new filter when we get this method call? + // It probably implies a new pruned job completed, and these TX would be checked more often then the older ones. + // However, we have less filters to use/faster filter deletion + for (Hash transactionHash : transactionHashes) { + addTransaction(transactionHash); + } + try { + persistFilter(lastAddedFilter, highestIndex); + } catch (Exception e) { + throw new PrunedTransactionException(e); + } + } + + private CuckooFilter newFilter() throws Exception { + if (filters.isAtFullCapacity()) { + log.debug("Removing " + filters.peek()); + persistenceProvider.delete(Cuckoo.class, new IntegerIndex(getLowestIndex())); + } + + highestIndex++; + + // We keep a reference to the last filter to prevent looking it up every time + filters.offer(lastAddedFilter = new CuckooFilterImpl(filterSize, BUCKET_SIZE, FINGER_PRINT_SIZE)); + return lastAddedFilter; + } + + private boolean shouldSwitchFilter() { + // Cuckoo filter speed/accuracy is best when filters stay partially empty + return lastAddedFilter != null && lastAddedFilter.size() > filterSize; + } + + private int getLowestIndex() { + if (highestIndex < MAX_FILTERS) { + return 0; + } + + return highestIndex - MAX_FILTERS; + } +} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java index 04795c272b..c98efbd5a4 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java @@ -4,6 +4,7 @@ import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.service.transactionpruning.PrunedTransactionProvider; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; @@ -49,6 +50,11 @@ public abstract class AbstractTransactionPrunerJob implements TransactionPrunerJ */ private Snapshot snapshot; + /** + * Holds a reference to the provider of pruned transactions, which we will use for speeding up old references checks + */ + private PrunedTransactionProvider prunedTransactionProvider; + /** * {@inheritDoc} */ @@ -138,7 +144,23 @@ public TransactionPrunerJobStatus getStatus() { public void setStatus(TransactionPrunerJobStatus status) { this.status = status; } - + + /** + * {@inheritDoc} + */ + @Override + public PrunedTransactionProvider getPrunedProvider() { + return prunedTransactionProvider; + } + + /** + * {@inheritDoc} + */ + @Override + public void setPrunedProvider(PrunedTransactionProvider prunedTransactionProvider) { + this.prunedTransactionProvider = prunedTransactionProvider; + } + /** * {@inheritDoc} */ diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index a32c342286..b507137e76 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -240,6 +241,10 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { }); getTangle().deleteBatch(elementsToDelete); + getPrunedProvider().addTransactionBatch(elementsToDelete + .stream() + .map(a -> (Hash) a.low) + .collect(Collectors.toList())); } catch(Exception e) { throw new TransactionPruningException("failed to cleanup milestone #" + getCurrentIndex(), e); } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java index 780d078267..c29fec8994 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java @@ -4,7 +4,6 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; import com.iota.iri.model.persistables.Transaction; -import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.storage.Indexable; diff --git a/src/main/java/com/iota/iri/utils/Serializer.java b/src/main/java/com/iota/iri/utils/Serializer.java index da14da5380..967e809468 100644 --- a/src/main/java/com/iota/iri/utils/Serializer.java +++ b/src/main/java/com/iota/iri/utils/Serializer.java @@ -37,14 +37,40 @@ public static long getLong(byte[] bytes, int start) { return res; } + /** + * Reads the default amount of bytes for an int(4) and turns it into an int. + * Starts at the beginning of the array + * + * @param bytes The bytes we use to make an int + * @return The created int, or 0 when bytes is null + */ public static int getInteger(byte[] bytes) { return getInteger(bytes, 0); } + + /** + * Reads the default amount of bytes for an int(4) and turns it into an int. + * + * @param bytes The bytes we use to make an int + * @param start The point in the array from which we start reading + * @return The created int, or 0 when bytes is null + */ public static int getInteger(byte[] bytes, int start) { + return getInteger(bytes, start, Integer.BYTES); + } + + /** + * Reads the bytes for the given length, starting at the starting point given. + * + * @param bytes The bytes we use to make an int + * @param start The point in the array from which we start reading + * @param length Amount of bytes to read + * @return The created int, or 0 when bytes is null + */ + public static int getInteger(byte[] bytes, int start, int length) { if(bytes == null) { return 0; } - int length = Integer.BYTES; int res = 0; for (int i=0; i< length;i++) { res |= (bytes[start + i] & 0xFFL) << ((length-i-1) * 8); diff --git a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java index 9807ff2119..e60c6aad0e 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java +++ b/src/main/java/com/iota/iri/utils/datastructure/CuckooFilter.java @@ -1,5 +1,7 @@ package com.iota.iri.utils.datastructure; +import java.util.BitSet; + /** * The Cuckoo Filter is a probabilistic data structure that supports fast set membership testing. * @@ -83,4 +85,10 @@ public interface CuckooFilter { * @return the amount of stored items */ int size(); + + /** + * This method returns a copy of all the bits that make up this filter + * @return The ilter bits + */ + BitSet getFilterData(); } diff --git a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java index 8efa4e2aae..83bd08840c 100644 --- a/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java +++ b/src/main/java/com/iota/iri/utils/datastructure/impl/CuckooFilterImpl.java @@ -11,6 +11,7 @@ * This class implements the basic contract of the {@link CuckooFilter}. */ public class CuckooFilterImpl implements CuckooFilter { + /** * The amount of times we try to kick elements when inserting before we consider the index to be too full. */ @@ -117,6 +118,38 @@ public CuckooFilterImpl(int itemCount, int bucketSize, int fingerPrintSize) thro cuckooFilterTable = new CuckooFilterTable(tableSize, bucketSize, fingerPrintSize); } + + /** + * Advanced constructor that allows for fine tuning of the desired filter. + * + * It first saves a reference to the hash function and then checks the parameters - the finger print size cannot + * be bigger than 128 bits because SHA1 generates 160 bits and we use 128 of that for the fingerprint and the rest + * for the index. + * + * After verifying that the passed in parameters are reasonable, we calculate the required size of the + * {@link CuckooFilterTable} by increasing the table size exponentially until we can fit the desired item count with + * a load factor of <= 0.955. Finally we create the {@link CuckooFilterTable} that will hold our data. + * + * NOTE: The actual size will be slightly bigger since the size has to be a power of 2 and take the optimal load + * factor of 0.955 into account. + * + * @param itemCount the minimum amount of items that should fit into the filter + * @param bucketSize the amount of items that can be stored in each bucket + * @param fingerPrintSize the amount of bits per fingerprint (it has to be bigger than 0 and smaller than 128) + * @param filterData The data this filter is initialized with. Must match with the parameters + * @throws IllegalArgumentException if the finger print size is too small or too big + * @throws InternalError if the SHA1 hashing function can not be found with this java version [should never happen] + */ + public CuckooFilterImpl(int itemCount, int bucketSize, int fingerPrintSize, BitSet filterData) throws IllegalArgumentException, + InternalError { + + this(itemCount, bucketSize, fingerPrintSize); + + if (cuckooFilterTable.data.size() != filterData.size()) { + throw new IllegalArgumentException("Filter data does not match filter parameters"); + } + cuckooFilterTable.data = BitSet.valueOf(filterData.toByteArray()); + } /** * {@inheritDoc} @@ -435,6 +468,11 @@ private BitSet generateFingerPrint(byte[] hash) throws IllegalArgumentException return BitSetUtils.convertByteArrayToBitSet(hash, 4, fingerPrintSize); } + @Override + public BitSet getFilterData() { + return (BitSet) cuckooFilterTable.data.clone(); + } + /** * Internal helper class to represent items that are stored in the filter. * diff --git a/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java new file mode 100644 index 0000000000..803e56ef91 --- /dev/null +++ b/src/test/java/com/iota/iri/service/transactionpruning/PrunedTransactionProviderImplTest.java @@ -0,0 +1,99 @@ +package com.iota.iri.service.transactionpruning; + +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import com.iota.iri.TransactionTestUtils; +import com.iota.iri.conf.SnapshotConfig; +import com.iota.iri.model.Hash; +import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl; + +public class PrunedTransactionProviderImplTest { + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + public SnapshotConfig config; + + @Rule + public final TemporaryFolder dbFolder = new TemporaryFolder(); + + @Rule + public final TemporaryFolder logFolder = new TemporaryFolder(); + + PrunedTransactionProviderImpl provider; + + @Before + public void setUp() throws PrunedTransactionException { + Mockito.when(config.getPrunedTransactionsDbPath()).thenReturn(dbFolder.getRoot().getAbsolutePath()); + Mockito.when(config.getPrunedTransactionsDbLogPath()).thenReturn(logFolder.getRoot().getAbsolutePath()); + + provider = new PrunedTransactionProviderImpl(10000); + provider.init(config); + } + + @After + public void tearDown() { + dbFolder.delete(); + } + + @Test + public void containsMarginalOkayTest() throws PrunedTransactionException { + int size = 10000; + int contains = 0; + + Hash[] hashes = new Hash[size]; + for (int i=0; i size*0.995); + } +}