Merge branch 'release-v1.6.1' into dev

# Conflicts:
#	src/main/java/com/iota/iri/Iota.java
#	src/main/java/com/iota/iri/service/milestone/impl/MilestoneServiceImpl.java

DOCKER.md

@@ -2,13 +2,13 @@
 
 Run the official iotaledger/iri container, passing the mandatory -p option:
 
-```docker run iotaledger/iri:v1.6.0-RELEASE -p 14265```
+```docker run iotaledger/iri:v1.6.1-RELEASE -p 14265```
 
 This will get your a running IRI with its API listening on port 14265, no neighbours and an empty database. The IRI Docker container by default expects data at /iri/data. Use the `-v` option of the `docker run` command to mount volumes so to have persistent data. You can also pass more command line options to the docker run command and those will be passed to IRI.
 
 If you want to use a iri.ini file with the docker container, supposing it's stored under /path/to/conf/iri.ini on your docker host, then pass `-v /path/to/conf:/iri/conf` and add -c /iri/conf/iri.ini as docker run arguments. So for example the `docker run` command above would become:
 
-```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RELEASE -p 14265 -c /iri/conf/iri.ini```
+```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.1-RELEASE -p 14265 -c /iri/conf/iri.ini```
 
 Please refer to the IRI documentation for further command line options and iri.ini options.
 
@@ -61,7 +61,7 @@ ExecStart=/usr/bin/docker run \
 -p 14265:14265 \
 -p 15600:15600 \
 -p 14600:14600/udp  \
-iotaledger/iri:v1.6.0-RELEASE \
+iotaledger/iri:v1.6.1-RELEASE \
 -p 14265 \
 --zmq-enabled \
 --testnet

changelog.txt

@@ -1,3 +1,22 @@
+1.6.1
+    - Fix: Db exists() method optimization (#1338)
+    - Fix: Do not persist pruner state (#1342)
+    - Fix: Reduce db access and cpu usage needed to persist spent addresses (#1332)
+    - Fix: Added a NULL_HASH check for genesis transaction (#1309)
+    - Feature: Fresh transactions to request (#1316)
+    - Fix: missing user-agent header in cors (#1319)
+    - Fix: Batch process spent addresses to avoid out of memory issues (#1314)
+    - Fix: added a buffer to LSManager (#1286)
+    - Fix: test existence of spent addresses db do not point to correct folder (#1305)
+    - Change: Convert from trytes to asciiString (#1302)
+    - Documentation: Fixed and added javadoc comments for existing classes (#1026)
+    - Fix: Creates rest endpoint for iotaconfig related settings as suggested (#1200)
+    - Fix: make storeMessage store transactions (#1186)
+    - Change: add a plugin to create reproducible builds. (#1194)
+    - Feature: Add configurable variables for spent addresses DB (#1274)
+    - Fix: Posting invalid json to IRI will result in detailed error message (#1202)
+    - Fix: dns reverse resolving issue for neighbors that are added with their IP. (#1255)
+
 1.6.0
    - Feat: added config variables for local snapshots (#981)
    - Feat: added a generic helper class for dag traversal (#982)

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.iota</groupId>
     <artifactId>iri</artifactId>
-    <version>1.6.0-RELEASE</version>
+    <version>1.6.1-RELEASE</version>
     <name>IRI</name>
     <description>IOTA Reference Implementation</description>
 

src/main/java/com/iota/iri/BundleValidator.java

@@ -55,7 +55,7 @@ public class BundleValidator {
      * If the bundle is invalid then an empty list will be returned.
      * @throws Exception if a persistence error occured
      */
-    public static List<List<TransactionViewModel>> validate(Tangle tangle, Snapshot initialSnapshot, Hash tailHash) throws Exception {
+    public List<List<TransactionViewModel>> validate(Tangle tangle, Snapshot initialSnapshot, Hash tailHash) throws Exception {
 
         TransactionViewModel tail = TransactionViewModel.fromHash(tangle, tailHash);
         if (tail.getCurrentIndex() != 0 || tail.getValidity() == -1) {
@@ -194,10 +194,10 @@ public static List<List<TransactionViewModel>> validate(Tangle tangle, Snapshot
     /**
      * Checks that the bundle's inputs and outputs are balanced.
      *
-     * @param transactionViewModels list of transactions that are in a bundle
+     * @param transactionViewModels collection of transactions that are in a bundle
      * @return {@code true} if balanced, {@code false} if unbalanced or {@code transactionViewModels} is empty
      */
-    public static boolean isInconsistent(List<TransactionViewModel> transactionViewModels) {
+    public static boolean isInconsistent(Collection<TransactionViewModel> transactionViewModels) {
         long value = 0;
         for (final TransactionViewModel bundleTransactionViewModel : transactionViewModels) {
             if (bundleTransactionViewModel.value() != 0) {

src/main/java/com/iota/iri/IRI.java

@@ -42,7 +42,7 @@ public class IRI {
 
     public static final String MAINNET_NAME = "IRI";
     public static final String TESTNET_NAME = "IRI Testnet";
-    public static final String VERSION = "1.6.0-RELEASE";
+    public static final String VERSION = "1.6.1-RELEASE";
 
     /**
      * The entry point of IRI.

src/main/java/com/iota/iri/Iota.java

@@ -95,6 +95,8 @@ public class Iota {
 
     public final TransactionRequesterWorkerImpl transactionRequesterWorker;
 
+    public final BundleValidator bundleValidator;
+
     public final Tangle tangle;
     public final TransactionValidator transactionValidator;
     public final TipsSolidifier tipsSolidifier;
@@ -136,6 +138,7 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh
         transactionRequesterWorker = new TransactionRequesterWorkerImpl();
 
         // legacy code
+        bundleValidator = new BundleValidator();
         tangle = new Tangle();
         tipsViewModel = new TipsViewModel();
         transactionRequester = new TransactionRequester(tangle, snapshotProvider);
@@ -198,22 +201,22 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx
         //because we check whether spent addresses data exists
         snapshotProvider.init(configuration);
         spentAddressesProvider.init(configuration);
-        spentAddressesService.init(tangle, snapshotProvider, spentAddressesProvider, configuration);
+        spentAddressesService.init(tangle, snapshotProvider, spentAddressesProvider, bundleValidator, configuration);
         snapshotService.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, configuration);
         if (localSnapshotManager != null) {
             localSnapshotManager.init(snapshotProvider, snapshotService, transactionPruner, configuration);
         }
-        milestoneService.init(tangle, snapshotProvider, snapshotService, configuration);
+        milestoneService.init(tangle, snapshotProvider, snapshotService, bundleValidator, configuration);
         latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier,
                 configuration);
         latestSolidMilestoneTracker.init(tangle, snapshotProvider, milestoneService, ledgerService,
                 latestMilestoneTracker);
         seenMilestonesRetriever.init(tangle, snapshotProvider, transactionRequester);
         milestoneSolidifier.init(snapshotProvider, transactionValidator);
-        ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService);
+        ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, spentAddressesService,
+                bundleValidator);
         if (transactionPruner != null) {
-            transactionPruner.init(tangle, snapshotProvider, spentAddressesService, tipsViewModel, configuration)
-                    .restoreState();
+            transactionPruner.init(tangle, snapshotProvider, spentAddressesService, tipsViewModel, configuration);
         }
         transactionRequesterWorker.init(tangle, transactionRequester, tipsViewModel, node);
     }

src/main/java/com/iota/iri/service/API.java

@@ -586,7 +586,7 @@ private AbstractResponse checkConsistencyStatement(List<String> transactionsList
                 state = false;
                 info = "tails are not solid (missing a referenced tx): " + transaction;
                 break;
-            } else if (BundleValidator.validate(instance.tangle, instance.snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() == 0) {
+            } else if (instance.bundleValidator.validate(instance.tangle, instance.snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() == 0) {
                 state = false;
                 info = "tails are not consistent (bundle is invalid): " + transaction;
                 break;

src/main/java/com/iota/iri/service/ledger/LedgerService.java

@@ -75,7 +75,8 @@ public interface LedgerService {
 
     /**
      * Generates the accumulated balance changes of the transactions that are directly or indirectly referenced by the
-     * given transaction relative to the referenced milestone.<br />
+     * given transaction relative to the referenced milestone. Also persists the spent addresses that were found in the
+     * process<br />
      * <br />
      * It simply iterates over all approvees that have not been confirmed yet and that have not been processed already
      * (by being part of the {@code visitedNonMilestoneSubtangleHashes} set) and collects their balance changes.<br />

src/main/java/com/iota/iri/service/ledger/impl/LedgerServiceImpl.java

@@ -8,10 +8,12 @@
 import com.iota.iri.service.ledger.LedgerException;
 import com.iota.iri.service.ledger.LedgerService;
 import com.iota.iri.service.milestone.MilestoneService;
+import com.iota.iri.service.snapshot.Snapshot;
 import com.iota.iri.service.snapshot.SnapshotException;
 import com.iota.iri.service.snapshot.SnapshotProvider;
 import com.iota.iri.service.snapshot.SnapshotService;
 import com.iota.iri.service.snapshot.impl.SnapshotStateDiffImpl;
+import com.iota.iri.service.spentaddresses.SpentAddressesService;
 import com.iota.iri.storage.Tangle;
 
 import java.util.*;
@@ -42,6 +44,10 @@ public class LedgerServiceImpl implements LedgerService {
      */
     private MilestoneService milestoneService;
 
+    private SpentAddressesService spentAddressesService;
+
+    private BundleValidator bundleValidator;
+
     /**
      * Initializes the instance and registers its dependencies.<br />
      * <br />
@@ -61,12 +67,15 @@ public class LedgerServiceImpl implements LedgerService {
      * @return the initialized instance itself to allow chaining
      */
     public LedgerServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SnapshotService snapshotService,
-            MilestoneService milestoneService) {
+            MilestoneService milestoneService, SpentAddressesService spentAddressesService,
+                                  BundleValidator bundleValidator) {
 
         this.tangle = tangle;
         this.snapshotProvider = snapshotProvider;
         this.snapshotService = snapshotService;
         this.milestoneService = milestoneService;
+        this.spentAddressesService = spentAddressesService;
+        this.bundleValidator = bundleValidator;
 
         return this;
     }
@@ -153,7 +162,9 @@ public Map<Hash, Long> generateBalanceDiff(Set<Hash> visitedTransactions, Hash s
         Map<Hash, Long> state = new HashMap<>();
         Set<Hash> countedTx = new HashSet<>();
 
-        snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet().forEach(solidEntryPointHash -> {
+        Snapshot initialSnapshot = snapshotProvider.getInitialSnapshot();
+        Map<Hash, Integer> solidEntryPoints = initialSnapshot.getSolidEntryPoints();
+        solidEntryPoints.keySet().forEach(solidEntryPointHash -> {
             visitedTransactions.add(solidEntryPointHash);
             countedTx.add(solidEntryPointHash);
         });
@@ -173,11 +184,16 @@ public Map<Hash, Long> generateBalanceDiff(Set<Hash> visitedTransactions, Hash s
                             if (transactionViewModel.getCurrentIndex() == 0) {
                                 boolean validBundle = false;
 
-                                final List<List<TransactionViewModel>> bundleTransactions = BundleValidator.validate(
+                                final List<List<TransactionViewModel>> bundleTransactions = bundleValidator.validate(
                                         tangle, snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash());
 
                                 for (final List<TransactionViewModel> bundleTransactionViewModels : bundleTransactions) {
 
+                                    //ISSUE 1008: generateBalanceDiff should be refactored so we don't have those hidden
+                                    // concerns
+                                    spentAddressesService
+                                            .persistValidatedSpentAddressesAsync(bundleTransactionViewModels);
+
                                     if (BundleValidator.isInconsistent(bundleTransactionViewModels)) {
                                         break;
                                     }
@@ -217,6 +233,7 @@ public Map<Hash, Long> generateBalanceDiff(Set<Hash> visitedTransactions, Hash s
         return state;
     }
 
+
     /**
      * Generates the {@link com.iota.iri.model.StateDiff} that belongs to the given milestone in the database and marks
      * all transactions that have been approved by the milestone accordingly by setting their {@code snapshotIndex}

src/main/java/com/iota/iri/service/milestone/impl/MilestoneServiceImpl.java

@@ -63,6 +63,8 @@ public class MilestoneServiceImpl implements MilestoneService {
      */
     private ConsensusConfig config;
 
+    private BundleValidator bundleValidator;
+
     /**
      * This method initializes the instance and registers its dependencies.<br />
      * <br />
@@ -81,11 +83,12 @@ public class MilestoneServiceImpl implements MilestoneService {
      * @return the initialized instance itself to allow chaining
      */
     public MilestoneServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SnapshotService snapshotService,
-            ConsensusConfig config) {
+            BundleValidator bundleValidator, ConsensusConfig config) {
 
         this.tangle = tangle;
         this.snapshotProvider = snapshotProvider;
         this.snapshotService = snapshotService;
+        this.bundleValidator = bundleValidator;
         this.config = config;
 
         return this;
@@ -165,7 +168,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM
                 return VALID;
             }
 
-            final List<List<TransactionViewModel>> bundleTransactions = BundleValidator.validate(tangle,
+            final List<List<TransactionViewModel>> bundleTransactions = bundleValidator.validate(tangle,
                     snapshotProvider.getInitialSnapshot(), transactionViewModel.getHash());
 
             if (bundleTransactions.isEmpty()) {

src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java

@@ -496,14 +496,6 @@ private void cleanupOldData(SnapshotConfig config, TransactionPruner transaction
     private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot newSnapshot, SnapshotConfig config)
             throws SnapshotException {
 
-        try {
-            spentAddressesService.persistSpentAddresses(snapshotProvider.getInitialSnapshot().getIndex(),
-                    newSnapshot.getIndex());
-
-        } catch (Exception e) {
-            throw new SnapshotException(e);
-        }
-
         snapshotProvider.writeSnapshotToDisk(newSnapshot, config.getLocalSnapshotsBasePath());
 
         snapshotProvider.getLatestSnapshot().lockWrite();

src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java

@@ -21,19 +21,19 @@ public interface SpentAddressesService {
      */
     boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesException;
 
-    /**
-     * Calculates and persists all spent addresses in between a range that were validly signed
-     * 
-     * @param fromMilestoneIndex the lower bound milestone index (inclusive)
-     * @param toMilestoneIndex the upper bound milestone index (exclusive)
-     * @throws Exception when anything went wrong whilst calculating.
-     */
-    void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws Exception;
 
     /**
      * Persist all the verifiable spent from a given list of transactions
      * @param transactions transactions to obtain spends from
      * @throws SpentAddressesException
      */
     void persistSpentAddresses(Collection<TransactionViewModel> transactions) throws SpentAddressesException;
+
+    /**
+     * Persists the spent addresses of all pre-verified valid transactions in an asynchronous manner
+     *
+     * @param transactions <b>Transactions that have their signatures verified beforehand</b>.
+     *                     Non spent transactions will be filtered out when persisting
+     */
+    void persistValidatedSpentAddressesAsync(Collection<TransactionViewModel> transactions);
 }

src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java

@@ -86,7 +86,7 @@ private void readSpentAddressesFromStream(InputStream in) throws SpentAddressesE
     @Override
     public boolean containsAddress(Hash addressHash) throws SpentAddressesException {
         try {
-            return ((SpentAddress) rocksDBPersistenceProvider.get(SpentAddress.class, addressHash)).exists();
+            return rocksDBPersistenceProvider.exists(SpentAddress.class, addressHash);
         } catch (Exception e) {
             throw new SpentAddressesException(e);
         }