Feature: Store pruned transaction hashes in a cuckoo filter

src/main/java/com/iota/iri/conf/BaseIotaConfig.java

@@ -5,7 +5,6 @@
 import com.beust.jcommander.ParameterException;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.iota.iri.IRI;
 import com.iota.iri.crypto.SpongeFactory;
 import com.iota.iri.model.Hash;
 import com.iota.iri.model.HashFactory;
@@ -111,7 +110,9 @@ public abstract class BaseIotaConfig implements IotaConfig {
     protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH;
     protected String spentAddressesDbPath = Defaults.SPENT_ADDRESSES_DB_PATH;
     protected String spentAddressesDbLogPath = Defaults.SPENT_ADDRESSES_DB_LOG_PATH;
-
+    protected String prunedTransactionsDbPath = Defaults.PRUNED_TRANSACTIONS_DB_PATH;
+    protected String prunedTransactionsDbLogPath = Defaults.PRUNED_TRANSACTIONS_DB_LOG_PATH;
+
     public BaseIotaConfig() {
         //empty constructor
     }
@@ -667,13 +668,35 @@ protected void setSpentAddressesDbPath(String spentAddressesDbPath) {
     public String getSpentAddressesDbLogPath() {
         return spentAddressesDbLogPath;
     }
-
+    
     @JsonProperty
     @Parameter(names = {"--spent-addresses-db-log-path"}, description = SnapshotConfig.Descriptions.SPENT_ADDRESSES_DB_LOG_PATH)
     protected void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) {
         this.spentAddressesDbLogPath = spentAddressesDbLogPath;
     }
-
+
+    @Override
+    public String getPrunedTransactionsDbLogPath() {
+        return prunedTransactionsDbLogPath;
+    }
+
+    @JsonProperty
+    @Parameter(names = {"--pruned-transactions-db-log-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_LOG_PATH)
+    protected void setPrunedTransactionDbLogPath(String prunedTransactionsDbLogPath) {
+        this.prunedTransactionsDbLogPath = prunedTransactionsDbLogPath;
+    }
+
+    @Override
+    public String getPrunedTransactionsDbPath() {
+        return prunedTransactionsDbPath;
+    }
+
+    @JsonProperty
+    @Parameter(names = {"--pruned-transactions-db-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_PATH)
+    protected void setPrunedTransactionsDbPath(String prunedTransactionsDbPath) {
+        this.prunedTransactionsDbPath = prunedTransactionsDbPath;
+    }
+
     /**
      * Checks if ZMQ is enabled.
      * @return true if zmqEnableTcp or zmqEnableIpc is set.
@@ -951,6 +974,9 @@ public interface Defaults {
         int LOCAL_SNAPSHOTS_DEPTH_MIN = 100;
         String SPENT_ADDRESSES_DB_PATH = "spent-addresses-db";
         String SPENT_ADDRESSES_DB_LOG_PATH = "spent-addresses-log";
+
+        String PRUNED_TRANSACTIONS_DB_LOG_PATH = "spent-addresses-db";
+        String PRUNED_TRANSACTIONS_DB_PATH = "spent-addresses-log";
 
         String LOCAL_SNAPSHOTS_BASE_PATH = "mainnet";
         String SNAPSHOT_FILE = "/snapshotMainnet.txt";

src/main/java/com/iota/iri/conf/SnapshotConfig.java

@@ -76,6 +76,16 @@ public interface SnapshotConfig extends Config {
      */
     String getSpentAddressesDbLogPath();
 
+    /**
+     * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_PATH}
+     */
+    String getPrunedTransactionsDbPath();
+
+    /**
+     * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_LOG_PATH}
+     */
+    String getPrunedTransactionsDbLogPath();
+
     interface Descriptions {
 
         String LOCAL_SNAPSHOTS_ENABLED = "Flag that determines if local snapshots are enabled.";
@@ -94,5 +104,8 @@ interface Descriptions {
                 "from previous epochs";
         String SPENT_ADDRESSES_DB_PATH = "The folder where the spent addresses DB saves its data.";
         String SPENT_ADDRESSES_DB_LOG_PATH = "The folder where the spent addresses DB saves its logs.";
+        String PRUNED_TRANSACTIONS_DB_PATH = "The folder where the pruned transactions DB saves its data.";
+        String PRUNED_TRANSACTIONS_DB_LOG_PATH = "The folder where the pruned transactions DB saves its logs.";
     }
+
 }

src/main/java/com/iota/iri/model/persistables/Cuckoo.java

@@ -0,0 +1,82 @@
+package com.iota.iri.model.persistables;
+
+import java.util.BitSet;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+import com.iota.iri.model.IntegerIndex;
+import com.iota.iri.storage.Persistable;
+import com.iota.iri.utils.Serializer;
+
+/**
+ * Persistable to manage the data we get as a result of pruning a milestone and its transactions
+ */
+public class Cuckoo implements Persistable {
+
+    /**
+     * The filter number it belonged to in previous cycles
+     */
+    public IntegerIndex filterId;
+
+    /**
+     * The bits that make up the CF
+     */
+    public BitSet filterBits;
+
+    /**
+     * 
+     * {@inheritDoc}
+     */
+    @Override
+    public byte[] bytes() {
+        byte[] num = filterId.bytes();
+        return ArrayUtils.addAll(num, filterBits.toByteArray());
+    }
+
+    /**
+     * Reads a CuckooBucket from the provided bytes.
+     * First 4 bytes are the bucket id, second 4 are the index inside that bucket
+     * Rest of the bytes is the bucket data
+     * 
+     * {@inheritDoc}
+     */
+    @Override
+    public void read(byte[] bytes) {
+        if(bytes != null) {
+            filterId = new IntegerIndex(Serializer.getInteger(bytes, 0));
+
+            short start = 4;
+            filterBits = new BitSet(bytes.length - start);
+            for (int i = start; i < bytes.length; i++) {
+                filterBits.set(i-start, bytes[i]);
+            }
+        }
+    }
+
+    /**
+     * 
+     * {@inheritDoc}
+     */
+    @Override
+    public byte[] metadata() {
+        return new byte[0];
+    }
+
+    /**
+     * 
+     * {@inheritDoc}
+     */
+    @Override
+    public void readMetadata(byte[] bytes) {
+    }
+
+    /**
+     * 
+     * {@inheritDoc}
+     */
+    @Override
+    public boolean merge() {
+        return false;
+    }
+
+}

src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java

@@ -54,8 +54,7 @@ public SpentAddressesProviderImpl init(SnapshotConfig config)
                     {{put("spent-addresses", SpentAddress.class);}}, null);
             this.rocksDBPersistenceProvider.init();
             readPreviousEpochsSpentAddresses();
-        }
-        catch (Exception e) {
+        } catch (Exception e) {
             throw new SpentAddressesException("There is a problem with accessing stored spent addresses", e);
         }
         return this;

src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionException.java

@@ -0,0 +1,38 @@
+package com.iota.iri.service.transactionpruning;
+
+/**
+ * This class is used to wrap exceptions that are specific to the pruned transaction persistance.
+ *
+ * It allows us to distinct between the different kinds of errors that can happen during the execution of the code.
+ */
+public class PrunedTransactionException extends Exception {
+    /**
+     * Constructor of the exception which allows us to provide a specific error message and the cause of the error.
+     *
+     * @param message reason why this error occurred
+     * @param cause wrapped exception that caused this error
+     */
+    public PrunedTransactionException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * Constructor of the exception which allows us to provide a specific error message without having an underlying
+     * cause.
+     *
+     * @param message reason why this error occurred
+     */
+    public PrunedTransactionException(String message) {
+        super(message);
+    }
+
+    /**
+     * Constructor of the exception which allows us to wrap the underlying cause of the error without providing a
+     * specific reason.
+     *
+     * @param cause wrapped exception that caused this error
+     */
+    public PrunedTransactionException(Throwable cause) {
+        super(cause);
+    }
+}

src/main/java/com/iota/iri/service/transactionpruning/PrunedTransactionProvider.java

@@ -0,0 +1,36 @@
+package com.iota.iri.service.transactionpruning;
+
+import java.util.Collection;
+
+import com.iota.iri.model.Hash;
+
+/**
+ * Find, mark and store pruned transactions
+ */
+public interface PrunedTransactionProvider {
+
+    /**
+     * Checks if this transactions has been pruned
+     * 
+     * @param transactionHash The transaction to check for
+     * @return <code>true</code> if it is, else <code>false</code>
+     * @throws PrunedTransactionException If the provider fails to check the transaction
+     */
+    boolean containsTransaction(Hash transactionHash) throws PrunedTransactionException;
+
+    /**
+     * Mark a transaction as spent.
+     * 
+     * @param transactionHash the transaction which we want to mark.
+     * @throws PrunedTransactionException If the provider fails to add the transaction
+     */
+    void addTransaction(Hash transactionHash) throws PrunedTransactionException;
+
+    /**
+     * Mark all transactions as pruned.
+     * 
+     * @param transactionHashes The transactions we want to mark
+     * @throws PrunedTransactionException If the provider fails to add a transaction
+     */
+    void addTransactionBatch(Collection<Hash> transactionHashes) throws PrunedTransactionException;
+}

src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java

@@ -121,6 +121,20 @@ public interface TransactionPrunerJob {
      * @param transactionPrunerJobStatus new execution status of the job
      */
     void setStatus(TransactionPrunerJobStatus transactionPrunerJobStatus);
+
+    /**
+     * Getter for the pruned transaction maintainer.
+     *
+     * @return pruned transaction maintainer of the job.
+     */
+    PrunedTransactionProvider getPrunedProvider();
+
+    /**
+     * Setter for the pruned transaction maintainer.
+     *
+     * @param prunedTransactionProvider pruned transaction maintainer of the job
+     */
+    void setPrunedProvider(PrunedTransactionProvider prunedTransactionProvider);
 
     /**
      * This method processes the cleanup job and performs the actual pruning.

src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java

@@ -5,6 +5,7 @@
 import com.iota.iri.service.snapshot.SnapshotProvider;
 import com.iota.iri.service.spentaddresses.SpentAddressesProvider;
 import com.iota.iri.service.spentaddresses.SpentAddressesService;
+import com.iota.iri.service.transactionpruning.PrunedTransactionProvider;
 import com.iota.iri.service.transactionpruning.TransactionPruner;
 import com.iota.iri.service.transactionpruning.TransactionPrunerJob;
 import com.iota.iri.service.transactionpruning.TransactionPruningException;
@@ -110,6 +111,11 @@ public class AsyncTransactionPruner implements TransactionPruner {
      */
     private final Map<Class<? extends TransactionPrunerJob>, JobQueue> jobQueues = new HashMap<>();
 
+    /**
+     * Provider for managing transactions we delete from the database in an optimized data structure
+     */
+    private PrunedTransactionProvider prunedTransactionProvider;
+
     /**
      * This method initializes the instance and registers its dependencies.<br />
      * <br />
@@ -158,6 +164,7 @@ public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvi
     @Override
     public void addJob(TransactionPrunerJob job) throws TransactionPruningException {
         job.setTransactionPruner(this);
+        job.setPrunedProvider(prunedTransactionProvider);
         job.setSpentAddressesService(spentAddressesService);
         job.setSpentAddressesProvider(spentAddressesProvider);
         job.setTangle(tangle);