From 8357f2d2f77673b911d5476c248779cb66fb3449 Mon Sep 17 00:00:00 2001 From: legacycode Date: Wed, 28 Nov 2018 11:45:41 +0100 Subject: [PATCH 01/63] Fix #1022: will return errormessage when unknown command is given. (#1036) * fix #1022: will return errormessage when unknown command is given. * added javadoc to shutdown method. * Replaced assertNotNull by assertThat for checking returned instance type. --- src/main/java/com/iota/iri/IXI.java | 20 ++++- src/test/java/com/iota/iri/IXITest.java | 103 ++++++++++++------------ 2 files changed, 69 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/iota/iri/IXI.java b/src/main/java/com/iota/iri/IXI.java index ef4b1239e6..94b7e9ff79 100644 --- a/src/main/java/com/iota/iri/IXI.java +++ b/src/main/java/com/iota/iri/IXI.java @@ -4,6 +4,7 @@ import com.google.gson.GsonBuilder; import com.iota.iri.service.CallableRequest; import com.iota.iri.service.dto.AbstractResponse; +import com.iota.iri.service.dto.ErrorResponse; import com.sun.nio.file.SensitivityWatchEventModifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,7 +20,10 @@ import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.time.Instant; -import java.util.*; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -183,6 +187,10 @@ private Path getPackagePath(Path modulePath) { } public AbstractResponse processCommand(final String command, Map request) { + if(command == null || command.isEmpty()) { + return ErrorResponse.create("Command can not be null or empty"); + } + Pattern pattern = Pattern.compile("^(.*)\\.(.*)$"); Matcher matcher = pattern.matcher(command); @@ -192,7 +200,7 @@ public AbstractResponse processCommand(final String command, Map return ixiMap.get(matcher.group(2)).call(request); } } - return null; + return ErrorResponse.create("Command [" + command + "] is unknown"); } private void loadModule(Path modulePath) { @@ -202,7 +210,7 @@ private void loadModule(Path modulePath) { log.info("No package.json found in {}", modulePath); return; } - final Map packageJson; + Map packageJson; Reader packageJsonReader; try { packageJsonReader = new FileReader(packageJsonPath.toFile()); @@ -275,6 +283,10 @@ private void detach(String moduleName) { ixiLifetime.remove(moduleName); } + /** + * Cleans up the environment, shutdown the dir watcher thread and wait till all running api calls are completed. + * @throws InterruptedException if directory watching thread was unexpected interrupted. + */ public void shutdown() throws InterruptedException { if(dirWatchThread != null) { shutdown = true; @@ -284,4 +296,4 @@ public void shutdown() throws InterruptedException { ixiLifetime.clear(); } } -} +} \ No newline at end of file diff --git a/src/test/java/com/iota/iri/IXITest.java b/src/test/java/com/iota/iri/IXITest.java index 3acf06ad5d..0efa3070a4 100644 --- a/src/test/java/com/iota/iri/IXITest.java +++ b/src/test/java/com/iota/iri/IXITest.java @@ -1,79 +1,82 @@ package com.iota.iri; import com.iota.iri.service.dto.AbstractResponse; -import com.iota.iri.service.dto.IXIResponse; +import com.iota.iri.service.dto.ErrorResponse; +import org.hamcrest.CoreMatchers; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * Unit tests for {@link IXI} + */ public class IXITest { - static TemporaryFolder ixiDir = new TemporaryFolder(); - static IXI ixi; + private static TemporaryFolder ixiDir = new TemporaryFolder(); + private static IXI ixi; + /** + * Create IXI temporary directory and start IXI. + * @throws Exception if temporary folder can not be created. + */ @BeforeClass public static void setUp() throws Exception { ixiDir.create(); ixi = new IXI(); - ixi.init(ixiDir.getRoot().getAbsolutePath().toString()); + ixi.init(ixiDir.getRoot().getAbsolutePath()); } + /** + * Shutdown IXI and delete temporary folder. + * @throws InterruptedException if directory watch thread was interrupted. + */ @AfterClass - public static void tearDown() throws Exception { + public static void tearDown() throws InterruptedException { ixi.shutdown(); ixiDir.delete(); } + /** + * If an command matches the command pattern, but is not valid, expect an unknown command error message. + */ @Test - public void init() throws Exception { - AbstractResponse response; - IXIResponse ixiResponse; - - /* - final String testJs = - "var Callable = Java.type(\"com.iota.iri.service.CallableRequest\");\n" + - "print(\"hello world\");\n" + - "var IXIResponse = Java.type(\"com.iota.iri.service.dto.IXIResponse\");\n" + - "API.put(\"getParser\", new Callable({\n" + - "call: function(req) {\n" + - "var IntArray = Java.type(\"int[]\");\n" + - "var out = new IntArray(Math.floor(Math.random()*9)+1);\n" + - "out[0] = 2;\n" + - "var r = IXIResponse.create({\n" + - "myArray: out,\n" + - "name: \"Foo\"\n" + - "});\n" + - "return r;\n" + - "}\n" + - "}));"; - final String testPackage = "{\"main\": \"index.js\"}"; - - - ixiDir.newFolder("test"); - try (OutputStream out = new BufferedOutputStream( - Files.newOutputStream(Paths.get(ixiDir.getRoot().toString(),"test", "index.js"), CREATE))) { - out.write(testJs.getBytes()); - } - try (OutputStream out = new BufferedOutputStream( - Files.newOutputStream(Paths.get(ixiDir.getRoot().toString(),"test", "package.json"), CREATE))) { - out.write(testPackage.getBytes()); - } - // Allow IXI to load the file - Map request = new HashMap<>(); - Thread.sleep(1000); - response = IXI.instance().processCommand("test.getParser", request); - - assertFalse(response instanceof ErrorResponse); - assertTrue(response instanceof IXIResponse); + public void processCommandError() { + AbstractResponse response = ixi.processCommand("testCommand.testSuffix", null); + assertThat("Wrong type of response", response, CoreMatchers.instanceOf(ErrorResponse.class)); + assertTrue("Wrong error message returned in response", response.toString().contains("Command [testCommand.testSuffix] is unknown")); + } - ixiResponse = ((IXIResponse) response); - assertNotNull(ixiResponse.getResponse()); - */ + /** + * If null is given as a command, expect a parameter check error message. + */ + @Test + public void processCommandNull() { + AbstractResponse response = ixi.processCommand(null, null); + assertThat("Wrong type of response", response, CoreMatchers.instanceOf(ErrorResponse.class)); + assertTrue("Wrong error message returned in response", response.toString().contains("Command can not be null or empty")); } + /** + * If an empty string is given as a command, expect a parameter check error message. + */ @Test - public void processCommand() throws Exception { + public void processCommandEmpty() { + AbstractResponse response = ixi.processCommand("", null); + assertThat("Wrong type of response", response, CoreMatchers.instanceOf(ErrorResponse.class)); + assertTrue("Wrong error message returned in response", response.toString().contains("Command can not be null or empty")); + } + /** + * If an does not match the command pattern, expect an unknown command error message. + */ + @Test + public void processCommandUnknown() { + AbstractResponse response = ixi.processCommand("unknown", null); + assertThat("Wrong type of response", response, CoreMatchers.instanceOf(ErrorResponse.class)); + assertTrue("Wrong error message returned in response", response.toString().contains("Command [unknown] is unknown")); } -} +} \ No newline at end of file From 05e875b370a0b97eb294aa39d6e58d2781fb9079 Mon Sep 17 00:00:00 2001 From: Alon Elmaliah Date: Wed, 28 Nov 2018 22:38:03 +0200 Subject: [PATCH 02/63] tweak tip selection documentation. (#1161) * tweak tip selection documentation. * Apply suggestions from code review Co-Authored-By: alon-e * add javadoc to Alpha --- .../tipselection/EntryPointSelector.java | 13 ++++++------- .../tipselection/RatingCalculator.java | 4 ++-- .../iri/service/tipselection/TailFinder.java | 9 ++------- .../iri/service/tipselection/TipSelector.java | 12 ++++++------ .../service/tipselection/WalkValidator.java | 2 +- .../iota/iri/service/tipselection/Walker.java | 4 ++-- .../impl/CumulativeWeightCalculator.java | 9 +++++++-- .../impl/EntryPointSelectorImpl.java | 11 ++++++++--- .../service/tipselection/impl/RatingOne.java | 1 + .../tipselection/impl/TailFinderImpl.java | 6 +++++- .../tipselection/impl/TipSelectorImpl.java | 17 +++++++++++++++-- .../tipselection/impl/WalkValidatorImpl.java | 10 ++++++++-- .../tipselection/impl/WalkerAlpha.java | 19 +++++++++++++++++++ 13 files changed, 82 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/iota/iri/service/tipselection/EntryPointSelector.java b/src/main/java/com/iota/iri/service/tipselection/EntryPointSelector.java index 5afcee7ec2..fa57d6acf2 100644 --- a/src/main/java/com/iota/iri/service/tipselection/EntryPointSelector.java +++ b/src/main/java/com/iota/iri/service/tipselection/EntryPointSelector.java @@ -1,25 +1,24 @@ package com.iota.iri.service.tipselection; import com.iota.iri.model.Hash; +import com.iota.iri.utils.collections.interfaces.UnIterableMap; /** - * Selects an entryPoint for tip selection. + * Selects an {@code entryPoint} for tip selection. *

- * this point is used as the starting point where - * the particle starts the random walk. + * this point is used as the starting point for {@link Walker#walk(Hash, UnIterableMap, WalkValidator)} *

*/ public interface EntryPointSelector { /** - *get an entryPoint for tip selection + *get an {@code entryPoint} for tip selection * - *Uses depth to determine the entry point for - *the random walk. + *Uses depth to determine the entry point for the random walk. * * @param depth Depth, in milestones. a notion of how deep to search for a good starting point. - * @return Entry point for walk method + * @return Entry point for walk method * @throws Exception If DB fails to retrieve transactions */ Hash getEntryPoint(int depth)throws Exception; diff --git a/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java b/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java index 6d35bd829e..a99f8b01b1 100644 --- a/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/RatingCalculator.java @@ -14,11 +14,11 @@ public interface RatingCalculator { * Rating calculator *

* Calculates the rating of all the transactions that reference - * a given entry point. + * a given {@code entryPoint}. *

* * @param entryPoint Transaction hash of a selected entry point. - * @return Map + * @return Map of ratings for each transaction that references entryPoint. * @throws Exception If DB fails to retrieve transactions */ diff --git a/src/main/java/com/iota/iri/service/tipselection/TailFinder.java b/src/main/java/com/iota/iri/service/tipselection/TailFinder.java index 2234338fbd..e3f34120da 100644 --- a/src/main/java/com/iota/iri/service/tipselection/TailFinder.java +++ b/src/main/java/com/iota/iri/service/tipselection/TailFinder.java @@ -11,15 +11,10 @@ @FunctionalInterface public interface TailFinder { /** - *Method for finding tails of bundles - * - *

- * This method is used to find a tail (current_index=0) of a bundle, - * given any transaction hash in the bundle. - *

+ *Method for finding a tail (current_index=0) of a bundle given any transaction in the associated bundle. * * @param hash The transaction hash of any transaction in the bundle. - * @return Hash of the tail transaction. + * @return Hash of the tail transaction, or {@code Empty} if the tail is not found. * @throws Exception If DB fails to retrieve transactions */ Optional findTail(Hash hash) throws Exception; diff --git a/src/main/java/com/iota/iri/service/tipselection/TipSelector.java b/src/main/java/com/iota/iri/service/tipselection/TipSelector.java index 63bf3006ed..5fa82c0900 100644 --- a/src/main/java/com/iota/iri/service/tipselection/TipSelector.java +++ b/src/main/java/com/iota/iri/service/tipselection/TipSelector.java @@ -6,23 +6,23 @@ import com.iota.iri.model.Hash; /** - * Selects tips to be approved + * Selects tips to be approved by a new transaction. */ public interface TipSelector { /** - * Method for finding tips + * Method for finding tips. * *

- * This method is used to find tips for approval given a depth, - * if reference is present then tips will also reference this transaction. + * This method is used to find tips for approval given a {@code depth}, + * if {@code reference} is present then tips will also reference this transaction. *

* - * @param depth The depth that the transactions will be found from. + * @param depth The depth that tip selection will start from. * @param reference An optional transaction hash to be referenced by tips. - * @return Transactions to approve + * @return Tips to approve * @throws Exception If DB fails to retrieve transactions */ List getTransactionsToApprove(int depth, Optional reference) throws Exception; diff --git a/src/main/java/com/iota/iri/service/tipselection/WalkValidator.java b/src/main/java/com/iota/iri/service/tipselection/WalkValidator.java index 595dd36354..17e419e699 100644 --- a/src/main/java/com/iota/iri/service/tipselection/WalkValidator.java +++ b/src/main/java/com/iota/iri/service/tipselection/WalkValidator.java @@ -15,7 +15,7 @@ public interface WalkValidator { *

* * @param transactionHash Transaction hash to validate consistency of. - * @return True iff tail is valid. + * @return true if tail is valid, false otherwise. * @throws Exception If Validation fails to execute */ boolean isValid(Hash transactionHash) throws Exception; diff --git a/src/main/java/com/iota/iri/service/tipselection/Walker.java b/src/main/java/com/iota/iri/service/tipselection/Walker.java index d882a8ca4b..a06d4d7e65 100644 --- a/src/main/java/com/iota/iri/service/tipselection/Walker.java +++ b/src/main/java/com/iota/iri/service/tipselection/Walker.java @@ -14,8 +14,8 @@ public interface Walker { /** * Walk algorithm *

- * Starts from given entry point to select valid transactions to be used - * as tips. It will output a valid transaction as a tip. + * Starts from a given {@code entryPoint} to select valid transactions to be used + * as tips. Returns a valid transaction as a tip. *

* * @param entryPoint Transaction hash to start walk from. diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java index 92bb854234..3573b126e3 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculator.java @@ -20,7 +20,8 @@ import java.util.*; /** - * Implementation of RatingCalculator that gives the cumulative for each transaction referencing entryPoint. + * Implementation of {@link RatingCalculator} that calculates the cumulative weight + * for each transaction referencing {@code entryPoint}.
* Used to create a weighted random walks. * * @see https://github.com/alongalky/iota-docs/blob/master/cumulative.md @@ -28,10 +29,14 @@ public class CumulativeWeightCalculator implements RatingCalculator{ private static final Logger log = LoggerFactory.getLogger(CumulativeWeightCalculator.class); - public static final int MAX_FUTURE_SET_SIZE = 5000; + private static final int MAX_FUTURE_SET_SIZE = 5000; public final Tangle tangle; + /** + * Constructor for Cumulative Weight Calculator + * @param tangle Tangle object which acts as a database interface + */ public CumulativeWeightCalculator(Tangle tangle) { this.tangle = tangle; } diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java b/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java index a617765b41..18a8ecdf17 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/EntryPointSelectorImpl.java @@ -7,15 +7,20 @@ import com.iota.iri.storage.Tangle; /** - * Implementation of EntryPointSelector that given a depth N, returns a N-deep milestone. - * Meaning milestone(latestSolid - depth) - * Used to as a starting point for the random walk. + * Implementation of {@link EntryPointSelector} that given a depth {@code N}, returns a N-deep milestone. + * Meaning milestone(latestSolid - depth) + * Used as a starting point for the random walk. */ public class EntryPointSelectorImpl implements EntryPointSelector { private final Tangle tangle; private final MilestoneTracker milestoneTracker; + /** + * Constructor for Entry Point Selector + * @param tangle Tangle object which acts as a database interface. + * @param milestoneTracker instance of the milestone tracker, used to get latest milestone. + */ public EntryPointSelectorImpl(Tangle tangle, MilestoneTracker milestoneTracker) { this.tangle = tangle; this.milestoneTracker = milestoneTracker; diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/RatingOne.java b/src/main/java/com/iota/iri/service/tipselection/impl/RatingOne.java index dd46c0f37a..d6bcbe1349 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/RatingOne.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/RatingOne.java @@ -22,6 +22,7 @@ public RatingOne(Tangle tangle) { this.tangle = tangle; } + @Override public UnIterableMap calculate(Hash entryPoint) throws Exception { UnIterableMap rating = new TransformingMap<>(null, null); diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java b/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java index 042c46dfd6..52c8ee0064 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java @@ -9,13 +9,17 @@ import java.util.Set; /** - * Implementation of TailFinder that given a transaction hash finds the tail of the associated bundle. + * Implementation of {@link TailFinder} that given a transaction hash finds the tail of the associated bundle. * */ public class TailFinderImpl implements TailFinder { private final Tangle tangle; + /** + * Constructor for Tail Finder + * @param tangle Tangle object which acts as a database interface + */ public TailFinderImpl(Tangle tangle) { this.tangle = tangle; } diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/TipSelectorImpl.java b/src/main/java/com/iota/iri/service/tipselection/impl/TipSelectorImpl.java index 4da343bd88..72082f1783 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/TipSelectorImpl.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/TipSelectorImpl.java @@ -21,8 +21,8 @@ */ public class TipSelectorImpl implements TipSelector { - public static final String REFERENCE_TRANSACTION_TOO_OLD = "reference transaction is too old"; - public static final String TIPS_NOT_CONSISTENT = "inconsistent tips pair selected"; + private static final String REFERENCE_TRANSACTION_TOO_OLD = "reference transaction is too old"; + private static final String TIPS_NOT_CONSISTENT = "inconsistent tips pair selected"; private final EntryPointSelector entryPointSelector; private final RatingCalculator ratingCalculator; @@ -33,6 +33,17 @@ public class TipSelectorImpl implements TipSelector { private final MilestoneTracker milestoneTracker; private final TipSelConfig config; + /** + * Constructor for Tip Selector. + * + * @param tangle Tangle object which acts as a database interface. + * @param ledgerValidator instance of the ledger validator, used by walk validator to check ledger consistency. + * @param entryPointSelector instance of the entry point selector to get tip selection starting points. + * @param ratingCalculator instance of rating calculator, to calculate weighted walks. + * @param walkerAlpha instance of walker (alpha), to perform weighted random walks as per the IOTA white paper. + * @param milestoneTracker instance of the milestone tracker, used by walk validator to check ledger consistency. + * @param config configurations to set internal parameters. + */ public TipSelectorImpl(Tangle tangle, LedgerValidator ledgerValidator, EntryPointSelector entryPointSelector, @@ -54,6 +65,8 @@ public TipSelectorImpl(Tangle tangle, } /** + * {@inheritDoc} + * * Implementation of getTransactionsToApprove * * General process: diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/WalkValidatorImpl.java b/src/main/java/com/iota/iri/service/tipselection/impl/WalkValidatorImpl.java index 6ffd0753f0..bd933a9fbb 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/WalkValidatorImpl.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/WalkValidatorImpl.java @@ -13,7 +13,7 @@ import java.util.*; /** - * Implementation of WalkValidator that checks consistency of the ledger as part of validity checks. + * Implementation of {@link WalkValidator} that checks consistency of the ledger as part of validity checks. * * A transaction is only valid if: *
    @@ -32,11 +32,17 @@ public class WalkValidatorImpl implements WalkValidator { private final MilestoneTracker milestoneTracker; private final TipSelConfig config; - private Set maxDepthOkMemoization; private Map myDiff; private Set myApprovedHashes; + /** + * Constructor of Walk Validator + * @param tangle Tangle object which acts as a database interface. + * @param ledgerValidator instance of the ledger validator, used to check ledger consistency. + * @param milestoneTracker instance of the milestone tracker, used by walk validator to check ledger consistency. + * @param config configurations to set internal parameters. + */ public WalkValidatorImpl(Tangle tangle, LedgerValidator ledgerValidator, MilestoneTracker milestoneTracker, TipSelConfig config) { this.tangle = tangle; this.ledgerValidator = ledgerValidator; diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/WalkerAlpha.java b/src/main/java/com/iota/iri/service/tipselection/impl/WalkerAlpha.java index 2ce445f1bf..982b4b2c75 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/WalkerAlpha.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/WalkerAlpha.java @@ -23,6 +23,10 @@ */ public class WalkerAlpha implements Walker { + /** + * {@code alpha}: a positive number that controls the randomness of the walk. + * The closer it is to 0, the less bias the random walk will be. + */ private double alpha; private final Random random; @@ -32,6 +36,15 @@ public class WalkerAlpha implements Walker { private final TailFinder tailFinder; + /** + * Constructor for Walker Alpha. + * + * @param tailFinder instance of tailFinder, used to step from tail to tail in random walk. + * @param tangle Tangle object which acts as a database interface + * @param messageQ ZMQ handle to publish telemetrics. + * @param random a source of randomness. + * @param config configurations to set internal parameters. + */ public WalkerAlpha(TailFinder tailFinder, Tangle tangle, MessageQ messageQ, Random random, TipSelConfig config) { this.tangle = tangle; this.messageQ = messageQ; @@ -40,10 +53,16 @@ public WalkerAlpha(TailFinder tailFinder, Tangle tangle, MessageQ messageQ, Rand this.alpha = config.getAlpha(); } + /** + * @return {@link WalkerAlpha#alpha} + */ public double getAlpha() { return alpha; } + /** + * @param alpha {@link WalkerAlpha#alpha} + */ public void setAlpha(double alpha) { this.alpha = alpha; } From 5883633a06312602c4a2439906d7ade49ed7f2f4 Mon Sep 17 00:00:00 2001 From: Dyrell Chapman Date: Sat, 1 Dec 2018 12:58:33 -0700 Subject: [PATCH 03/63] Document Controllers (#1154) * Controllers initial commit * Add Hashes View Model Documentation * Add Address View Model Documentation * Add Transaction View Model Documentation pt 1 * Transaction View Model Documentation pt. 2 * Transaction View Model Documentation pt. 3 * Add Approvee View Model Documentation * Add Bundle View Model Documentation * Update javadoc inline annotation * Add Tag View Model Documentation * Tips View Model Documentation pt. 1 * Tips View Model Documentation pt. 2 * Add StateDiff View Model Documentation * Add Milestone View Model Documentation * Update Transaction View Model Documentation * Overrides for store impl. * Rework doc inheritence and organisation * Reorganise Transaction View Model Documentation * Remove doc redundancies * Apply suggestions from code review Update setSolid doc Co-Authored-By: DyrellC --- .../iri/controllers/AddressViewModel.java | 54 +- .../iri/controllers/ApproveeViewModel.java | 57 +- .../iota/iri/controllers/BundleViewModel.java | 67 ++- .../iota/iri/controllers/HashesViewModel.java | 46 ++ .../iri/controllers/MilestoneViewModel.java | 95 ++++ .../iri/controllers/StateDiffViewModel.java | 56 +- .../iota/iri/controllers/TagViewModel.java | 86 ++- .../iota/iri/controllers/TipsViewModel.java | 88 ++- .../iri/controllers/TransactionViewModel.java | 501 +++++++++++++++--- 9 files changed, 915 insertions(+), 135 deletions(-) diff --git a/src/main/java/com/iota/iri/controllers/AddressViewModel.java b/src/main/java/com/iota/iri/controllers/AddressViewModel.java index 2237d7c475..b25aaa5fa5 100644 --- a/src/main/java/com/iota/iri/controllers/AddressViewModel.java +++ b/src/main/java/com/iota/iri/controllers/AddressViewModel.java @@ -10,57 +10,93 @@ import java.util.Set; /** - * Created by paul on 5/15/17. + * Acts as a controller interface for an {@link Address} set. This controller is used within a + * {@link TransactionViewModel} to manipulate an {@link Address} set. */ public class AddressViewModel implements HashesViewModel { private Address self; private Indexable hash; + /** + * Constructor for an {@link Address} set controller from a {@link Hash} identifier. + * @param hash The {@link Hash} identifier that the controller will be created for. + */ public AddressViewModel(Hash hash) { this.hash = hash; } + /** + * Constructor for an {@link Address} set controller from an existing {@link Address} set. If the set is empty, an + * {@link Address} set is created. + * + * @param hashes The {@link Address} set that the controller will be created from + * @param hash The {@link Hash} identifier that acts as a reference for the {@link Address} set + */ private AddressViewModel(Address hashes, Indexable hash) { self = hashes == null || hashes.set == null ? new Address(): hashes; this.hash = hash; } + /** + * Creates a new {@link Address} set controller. This controller is created by extracting the {@link Address} set + * from the database using the provided {@link Hash} identifier. + * + * @param tangle The tangle reference for the database to find the {@link Address} set in + * @param hash The hash identifier for the {@link Address} set that needs to be found + * @return The {@link AddressViewModel} controller generated + * @throws Exception Thrown if the database cannot load an {@link Address} set from the reference {@link Hash} + */ public static AddressViewModel load(Tangle tangle, Indexable hash) throws Exception { return new AddressViewModel((Address) tangle.load(Address.class, hash), hash); } + /** + * Fetches the first persistable {@link Address} set from the database and generates a new {@link AddressViewModel} + * from it. If no {@link Address} sets exist in the database, it will return null. + * + * @param tangle the tangle reference for the database + * @return The new {@link AddressViewModel} + * @throws Exception Thrown if the database fails to return a first object + */ + public static AddressViewModel first(Tangle tangle) throws Exception { + Pair bundlePair = tangle.getFirst(Address.class, Hash.class); + if(bundlePair != null && bundlePair.hi != null) { + return new AddressViewModel((Address) bundlePair.hi, (Hash) bundlePair.low); + } + return null; + } + + @Override public boolean store(Tangle tangle) throws Exception { return tangle.save(self, hash); } + @Override public int size() { return self.set.size(); } + @Override public boolean addHash(Hash theHash) { return getHashes().add(theHash); } + @Override public Indexable getIndex() { return hash; } + @Override public Set getHashes() { return self.set; } + @Override public void delete(Tangle tangle) throws Exception { tangle.delete(Address.class,hash); } - public static AddressViewModel first(Tangle tangle) throws Exception { - Pair bundlePair = tangle.getFirst(Address.class, Hash.class); - if(bundlePair != null && bundlePair.hi != null) { - return new AddressViewModel((Address) bundlePair.hi, (Hash) bundlePair.low); - } - return null; - } - + @Override public AddressViewModel next(Tangle tangle) throws Exception { Pair bundlePair = tangle.next(Address.class, hash); if(bundlePair != null && bundlePair.hi != null) { diff --git a/src/main/java/com/iota/iri/controllers/ApproveeViewModel.java b/src/main/java/com/iota/iri/controllers/ApproveeViewModel.java index a1e65fc555..28ce1f627e 100644 --- a/src/main/java/com/iota/iri/controllers/ApproveeViewModel.java +++ b/src/main/java/com/iota/iri/controllers/ApproveeViewModel.java @@ -11,61 +11,94 @@ import java.util.Map; import java.util.Set; +/** + * Acts as a controller interface for an {@link Approvee} set. This controller is used within a + * {@link TransactionViewModel} to manipulate an {@link Approvee} set. + */ public class ApproveeViewModel implements HashesViewModel { private Approvee self; private Indexable hash; + /** + * Constructor for an {@link Approvee} set controller from a {@link Hash} identifier. + * @param hash The {@link Hash} identifier that the controller will be created for. + */ public ApproveeViewModel(Hash hash) { this.hash = hash; } + /** + * Constructor for an {@link Approvee} set controller from an existing {@link Approvee} set. If the set is empty, a + * new {@link Approvee} set is created. + * + * @param hashes The {@link Approvee} set that the controller will be created from + * @param hash The {@link Hash} identifier that acts as a reference for the {@link Approvee} set + */ private ApproveeViewModel(Approvee hashes, Indexable hash) { self = hashes == null || hashes.set == null ? new Approvee(): hashes; this.hash = hash; } + /** + * Creates a new {@link Approvee} set controller. This controller is created by extracting the {@link Approvee} set + * from the database using the provided {@link Hash} identifier. + * + * @param tangle The tangle reference for the database to find the {@link Approvee} set in + * @param hash The hash identifier for the {@link Approvee} set that needs to be found + * @return The {@link ApproveeViewModel} controller generated + * @throws Exception Thrown if the database cannot load an {@link Approvee} set from the reference {@link Hash} + */ public static ApproveeViewModel load(Tangle tangle, Indexable hash) throws Exception { return new ApproveeViewModel((Approvee) tangle.load(Approvee.class, hash), hash); } - public static Map.Entry getEntry(Hash hash, Hash hashToMerge) throws Exception { - Approvee hashes = new Approvee(); - hashes.set.add(hashToMerge); - return new HashMap.SimpleEntry<>(hash, hashes); + /** + * Fetches the first persistable {@link Approvee} set from the database and generates a new + * {@link ApproveeViewModel} from it. If no {@link Approvee} sets exist in the database, it will return null. + * + * @param tangle the tangle reference for the database + * @return The new {@link ApproveeViewModel} + * @throws Exception Thrown if the database fails to return a first object + */ + public static ApproveeViewModel first(Tangle tangle) throws Exception { + Pair bundlePair = tangle.getFirst(Approvee.class, Hash.class); + if(bundlePair != null && bundlePair.hi != null) { + return new ApproveeViewModel((Approvee) bundlePair.hi, (Hash) bundlePair.low); + } + return null; } + @Override public boolean store(Tangle tangle) throws Exception { return tangle.save(self, hash); } + @Override public int size() { return self.set.size(); } + @Override public boolean addHash(Hash theHash) { return getHashes().add(theHash); } + @Override public Indexable getIndex() { return hash; } + @Override public Set getHashes() { return self.set; } + @Override public void delete(Tangle tangle) throws Exception { tangle.delete(Approvee.class,hash); } - public static ApproveeViewModel first(Tangle tangle) throws Exception { - Pair bundlePair = tangle.getFirst(Approvee.class, Hash.class); - if(bundlePair != null && bundlePair.hi != null) { - return new ApproveeViewModel((Approvee) bundlePair.hi, (Hash) bundlePair.low); - } - return null; - } - + @Override public ApproveeViewModel next(Tangle tangle) throws Exception { Pair bundlePair = tangle.next(Approvee.class, hash); if(bundlePair != null && bundlePair.hi != null) { diff --git a/src/main/java/com/iota/iri/controllers/BundleViewModel.java b/src/main/java/com/iota/iri/controllers/BundleViewModel.java index bc5daf1239..2b32bad82c 100644 --- a/src/main/java/com/iota/iri/controllers/BundleViewModel.java +++ b/src/main/java/com/iota/iri/controllers/BundleViewModel.java @@ -11,53 +11,87 @@ import java.util.Map; import java.util.Set; +/** + * Acts as a controller interface for a {@link Bundle} set. This controller is used within a + * {@link TransactionViewModel} to manipulate a {@link Bundle} set. + */ public class BundleViewModel implements HashesViewModel { private Bundle self; private Indexable hash; + /** + * Constructor for a {@link Bundle} set controller from a {@link Hash} identifier. + * @param hash The {@link Hash} identifier that the controller will be created for. + */ public BundleViewModel(Hash hash) { this.hash = hash; } + /** + * Constructor for a {@link Bundle} set controller from an existing {@link Bundle} set. If the set is empty, a new + * {@link Bundle} set is created. + * + * @param hashes The {@link Bundle} set that the controller will be created from + * @param hash The {@link Hash} identifier that acts as a reference for the {@link Bundle} set + */ private BundleViewModel(Bundle hashes, Indexable hash) { self = hashes == null || hashes.set == null ? new Bundle(): hashes; this.hash = hash; } + /** + * Creates a new {@link Bundle} set controller. This controller is created by extracting the {@link Bundle} set + * from the database using the provided {@link Hash} identifier. + * + * @param tangle The tangle reference for the database to find the {@link Bundle} set in + * @param hash The hash identifier for the {@link Bundle} set that needs to be found + * @return The {@link BundleViewModel} controller generated + * @throws Exception Thrown if the database cannot load an {@link Bundle} set from the reference {@link Hash} + */ public static BundleViewModel load(Tangle tangle, Indexable hash) throws Exception { return new BundleViewModel((Bundle) tangle.load(Bundle.class, hash), hash); } - public static Map.Entry getEntry(Hash hash, Hash hashToMerge) throws Exception { - Bundle hashes = new Bundle(); - hashes.set.add(hashToMerge); - return new HashMap.SimpleEntry<>(hash, hashes); - } - - /* - public static boolean merge(Hash hash, Hash hashToMerge) throws Exception { - Bundle hashes = new Bundle(); - hashes.set = new HashSet<>(Collections.singleton(hashToMerge)); - return Tangle.instance().merge(hashes, hash); + /** + * Fetches the first persistable {@link Bundle} set from the database and generates a new + * {@link BundleViewModel} from it. If no {@link Bundle} sets exist in the database, it will return null. + * + * @param tangle the tangle reference for the database + * @return The new {@link BundleViewModel} + * @throws Exception Thrown if the database fails to return a first object + */ + public static BundleViewModel first(Tangle tangle) throws Exception { + Pair bundlePair = tangle.getFirst(Bundle.class, Hash.class); + if(bundlePair != null && bundlePair.hi != null) { + return new BundleViewModel((Bundle) bundlePair.hi, (Hash) bundlePair.low); + } + return null; } - */ + /** + * {@inheritDoc} + */ + @Override public boolean store(Tangle tangle) throws Exception { return tangle.save(self, hash); } + @Override public int size() { return self.set.size(); } + @Override public boolean addHash(Hash theHash) { return getHashes().add(theHash); } + @Override public Indexable getIndex() { return hash; } + @Override public Set getHashes() { return self.set; } @@ -67,14 +101,7 @@ public void delete(Tangle tangle) throws Exception { tangle.delete(Bundle.class,hash); } - public static BundleViewModel first(Tangle tangle) throws Exception { - Pair bundlePair = tangle.getFirst(Bundle.class, Hash.class); - if(bundlePair != null && bundlePair.hi != null) { - return new BundleViewModel((Bundle) bundlePair.hi, (Hash) bundlePair.low); - } - return null; - } - + @Override public BundleViewModel next(Tangle tangle) throws Exception { Pair bundlePair = tangle.next(Bundle.class, hash); if(bundlePair != null && bundlePair.hi != null) { diff --git a/src/main/java/com/iota/iri/controllers/HashesViewModel.java b/src/main/java/com/iota/iri/controllers/HashesViewModel.java index 61c1b4da1d..cabc0e4629 100644 --- a/src/main/java/com/iota/iri/controllers/HashesViewModel.java +++ b/src/main/java/com/iota/iri/controllers/HashesViewModel.java @@ -6,13 +6,59 @@ import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Tangle; +/** + * Base implementation of a controller interface for sets of {@link com.iota.iri.model.persistables.Hashes}. + */ public interface HashesViewModel { + + /** + * Store the {@link com.iota.iri.model.persistables.Hashes} set and {@link Hash} reference to the database + * + * @param tangle The tangle reference for the database + * @return True if the object was saved correctly, False if not + * @throws Exception Thrown if the {@link com.iota.iri.model.persistables.Hashes} set or index {@link Hash} are null + */ boolean store(Tangle tangle) throws Exception; + + /** + * @return The size of the {@link com.iota.iri.model.persistables.Hashes} set referenced by the controller + */ int size(); + + /** + * Add a hash object to the controllers referenced {@link com.iota.iri.model.persistables.Hashes} set + * + * @param theHash The {@link Hash} identifier to be added to the set + * @return True if the {@link com.iota.iri.model.persistables.Hashes} set is added correctly, False if not + */ boolean addHash(Hash theHash); + + /** + * @return The {@link Hash} identifier of the {@link com.iota.iri.model.persistables.Hashes} set + */ Indexable getIndex(); + + /** + * @return The {@link com.iota.iri.model.persistables.Hashes} set referenced by the controller + */ Set getHashes(); + + /** + * Deletes a referenced {@link com.iota.iri.model.persistables.Hashes} set from the database + * + * @param tangle The tangle reference for the database + * @throws Exception If the {@link com.iota.iri.model.persistables.Hashes} set does not exist or fails to be removed + */ void delete(Tangle tangle) throws Exception; + /** + * Fetches the next indexed persistable {@link com.iota.iri.model.persistables.Hashes} set from the database and + * generates a new {@link HashesViewModel} from it. If no {@link com.iota.iri.model.persistables.Hashes} sets exist + * in the database, it will return null. + * + * @param tangle The tangle reference for the database + * @return The new {@link HashesViewModel} + * @throws Exception If the database fails to return a next {@link com.iota.iri.model.persistables.Hashes} set + */ HashesViewModel next(Tangle tangle) throws Exception; } diff --git a/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java b/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java index 55723f7163..9f35a9724e 100644 --- a/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java +++ b/src/main/java/com/iota/iri/controllers/MilestoneViewModel.java @@ -11,6 +11,10 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +/** + * Acts as a controller interface for a {@link Milestone} hash object. This controller is used by the + * {@link com.iota.iri.MilestoneTracker} to manipulate a {@link Milestone} object. + */ public class MilestoneViewModel { private final Milestone milestone; private static final Map milestones = new ConcurrentHashMap<>(); @@ -19,6 +23,9 @@ private MilestoneViewModel(final Milestone milestone) { this.milestone = milestone; } + /** + * Removes the contents of the stored {@link Milestone} object set. + */ public static void clear() { milestones.clear(); } @@ -35,12 +42,29 @@ public static void clear(int milestoneIndex) { milestones.remove(milestoneIndex); } + /** + * Constructor for a {@link Milestone} set controller. This controller is generated from a finalized + * {@link Milestone} hash identifier, indexing this object to the integer {@link Milestone} index. + * + * @param index The finalized numerical index the {@link Milestone} object will be referenced by in the set + * @param milestoneHash The finalized {@link Hash} identifier for the {@link Milestone} object + */ public MilestoneViewModel(final int index, final Hash milestoneHash) { this.milestone = new Milestone(); this.milestone.index = new IntegerIndex(index); milestone.hash = milestoneHash; } + /** + * Fetches an existing {@link MilestoneViewModel} if its index reference can be found in the controller. If the + * {@link MilestoneViewModel} is null, but the indexed {@link Milestone} object exists in the database, a new + * controller is created for the {@link Milestone} object. + * + * @param tangle The tangle reference for the database + * @param index The integer index of the {@link Milestone} object that the controller should be returned for + * @return The {@link MilestoneViewModel} for the indexed {@link Milestone} object + * @throws Exception Thrown if the database fails to load the indexed {@link Milestone} object + */ public static MilestoneViewModel get(Tangle tangle, int index) throws Exception { MilestoneViewModel milestoneViewModel = milestones.get(index); if(milestoneViewModel == null && load(tangle, index)) { @@ -49,6 +73,17 @@ public static MilestoneViewModel get(Tangle tangle, int index) throws Exception return milestoneViewModel; } + /** + * Fetches a {@link Milestone} object from the database using its integer index. If the {@link Milestone} and the + * associated {@link Hash} identifier are not null, a new {@link MilestoneViewModel} is created for the + * {@link Milestone} object, and it is placed into the Milestones set, indexed by the provided integer + * index. + * + * @param tangle The tangle reference for the database + * @param index The integer index reference for the {@link Milestone} object + * @return True if the {@link Milestone} object is stored in the Milestones set, False if not + * @throws Exception Thrown if the database fails to load the {@link Milestone} object + */ public static boolean load(Tangle tangle, int index) throws Exception { Milestone milestone = (Milestone) tangle.load(Milestone.class, new IntegerIndex(index)); if(milestone != null && milestone.hash != null) { @@ -58,6 +93,14 @@ public static boolean load(Tangle tangle, int index) throws Exception { return false; } + /** + * Fetches the first persistable {@link Milestone} object from the database and generates a new + * {@link MilestoneViewModel} from it. If no {@link Milestone} objects exist in the database, it will return null. + * + * @param tangle the tangle reference for the database + * @return The new {@link MilestoneViewModel} + * @throws Exception Thrown if the database fails to return a first object + */ public static MilestoneViewModel first(Tangle tangle) throws Exception { Pair milestonePair = tangle.getFirst(Milestone.class, IntegerIndex.class); if(milestonePair != null && milestonePair.hi != null) { @@ -67,6 +110,14 @@ public static MilestoneViewModel first(Tangle tangle) throws Exception { return null; } + /** + * Fetches the most recent persistable {@link Milestone} object from the database and generates a new + * {@link MilestoneViewModel} from it. If no {@link Milestone} objects exist in the database, it will return null. + * + * @param tangle the tangle reference for the database + * @return The new {@link MilestoneViewModel} + * @throws Exception Thrown if the database fails to return a first object + */ public static MilestoneViewModel latest(Tangle tangle) throws Exception { Pair milestonePair = tangle.getLatest(Milestone.class, IntegerIndex.class); if(milestonePair != null && milestonePair.hi != null) { @@ -76,6 +127,15 @@ public static MilestoneViewModel latest(Tangle tangle) throws Exception { return null; } + + /** + * Fetches the previously indexed persistable {@link Milestone} object from the database and generates a new + * {@link MilestoneViewModel} from it. If no {@link Milestone} objects exist in the database, it will return null. + * + * @param tangle the tangle reference for the database + * @return The new {@link MilestoneViewModel} + * @throws Exception Thrown if the database fails to return a first object + */ public MilestoneViewModel previous(Tangle tangle) throws Exception { Pair milestonePair = tangle.previous(Milestone.class, this.milestone.index); if(milestonePair != null && milestonePair.hi != null) { @@ -85,6 +145,14 @@ public MilestoneViewModel previous(Tangle tangle) throws Exception { return null; } + /** + * Fetches the next indexed persistable {@link Milestone} object from the database and generates a new + * {@link MilestoneViewModel} from it. If no {@link Milestone} objects exist in the database, it will return null. + * + * @param tangle The tangle reference for the database + * @return The new {@link MilestoneViewModel} + * @throws Exception Thrown if the database fails to return a first object + */ public MilestoneViewModel next(Tangle tangle) throws Exception { Pair milestonePair = tangle.next(Milestone.class, this.milestone.index); if(milestonePair != null && milestonePair.hi != null) { @@ -94,6 +162,17 @@ public MilestoneViewModel next(Tangle tangle) throws Exception { return null; } + /** + * Fetches a {@link MilestoneViewModel} for the closest {@link Milestone} object previously indexed in the + * database. The method starts at the provided index and works backwards through the database to try and find a + * {@link MilestoneViewModel} for the previous indexes until a non null controller is found. + * + * @param tangle The tangle reference for the database + * @param index The beginning index the method will work backwards from + * @param minIndex The minimum index that should be found in the database + * @return The {@link MilestoneViewModel} of the closest found controller previously indexed in the database + * @throws Exception Thrown if there is a failure to fetch a previous {@link MilestoneViewModel} + */ public static MilestoneViewModel findClosestPrevMilestone(Tangle tangle, int index, int minIndex) throws Exception { // search for the previous milestone preceding our index MilestoneViewModel previousMilestoneViewModel = null; @@ -128,17 +207,33 @@ public static MilestoneViewModel findClosestNextMilestone(Tangle tangle, int ind return nextMilestoneViewModel; } + /** + * Save the {@link Milestone} object, indexed by its integer index, to the database. + * + * @param tangle The tangle reference for the database + * @return True if the {@link Milestone} object is saved correctly, False if not + * @throws Exception Thrown if there is an error while saving the {@link Milestone} object + */ public boolean store(Tangle tangle) throws Exception { return tangle.save(milestone, milestone.index); } + /**@return The {@link Hash} identifier of the {@link Milestone} object*/ public Hash getHash() { return milestone.hash; } + + /**@return The integer index of the {@link Milestone} object*/ public Integer index() { return milestone.index.getValue(); } + /** + * Removes the {@link Milestone} object from the database. + * + * @param tangle The tangle reference for the database + * @throws Exception Thrown if there is an error removing the {@link Milestone} object + */ public void delete(Tangle tangle) throws Exception { tangle.delete(Milestone.class, milestone.index); } diff --git a/src/main/java/com/iota/iri/controllers/StateDiffViewModel.java b/src/main/java/com/iota/iri/controllers/StateDiffViewModel.java index 56645b17d9..807d625cf4 100644 --- a/src/main/java/com/iota/iri/controllers/StateDiffViewModel.java +++ b/src/main/java/com/iota/iri/controllers/StateDiffViewModel.java @@ -6,46 +6,100 @@ import java.util.Map; +/** + * Acts as a controller interface for a {@link StateDiff}. This controller is used to manipulate a {@link StateDiff} + * mapping of {@link com.iota.iri.model.persistables.Hashes} to Balance states in the database. + */ public class StateDiffViewModel { private StateDiff stateDiff; private Hash hash; + /** + * Creates a {@link StateDiff} controller using a {@link Hash} identifier as a reference point. A {@link StateDiff} + * is loaded from the database using the {@link Hash} reference, and the {@link Hash} identifier is set as the + * controller reference as well. + * + * @param tangle The tangle reference for the database + * @param hash The {@link Hash} identifier of the {@link StateDiff} the controller will be created for + * @return The new {@link StateDiffViewModel} + * @throws Exception Thrown if there is an error loading the {@link StateDiff} from the database + */ public static StateDiffViewModel load(Tangle tangle, Hash hash) throws Exception { return new StateDiffViewModel((StateDiff) tangle.load(StateDiff.class, hash), hash); } + /** + * Constructor for a {@link StateDiff} controller using a predefined {@link StateDiff} mapping. The {@link Hash} + * identifier is assigned as a reference for the controller, and the state is stored in the controller. + * + * @param state The {@link StateDiff} mapping that the controller will be made for + * @param hash The reference {@link Hash} identifier + */ public StateDiffViewModel(final Map state, final Hash hash) { this.hash = hash; this.stateDiff = new StateDiff(); this.stateDiff.state = state; } + /** + * This method checks the {@link com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider} to determine if an object + * might exist in the database. If it definitively does not exist, it will return False + * + * @param tangle The tangle reference for the database + * @param hash The {@link Hash} identifier of the object you are looking for + * @return True if the key might exist in the database, False if it definitively does not + * @throws Exception Thrown if there is an error checking the database + */ public static boolean maybeExists(Tangle tangle, Hash hash) throws Exception { return tangle.maybeHas(StateDiff.class, hash); } - StateDiffViewModel(final StateDiff diff, final Hash hash) { + /** + * Creates a finalized {@link StateDiff} controller. The referenced {@link StateDiff} of this controller and its + * reference {@link Hash} identifier cannot be modified. If the provided {@link StateDiff} is null, an empty + * {@link StateDiff} will be created. + * + * @param diff The finalized {@link StateDiff} the controller will be made for + * @param hash The finalized {@link Hash} identifier of the controller + */ + private StateDiffViewModel(final StateDiff diff, final Hash hash) { this.hash = hash; this.stateDiff = diff == null || diff.state == null ? new StateDiff(): diff; } + /**@return True if the {@link StateDiff} is empty, False if there is a variable present*/ public boolean isEmpty() { return stateDiff == null || stateDiff.state == null || stateDiff.state.size() == 0; } + /**@return The {@link Hash} identifier of the {@link StateDiff} controller */ public Hash getHash() { return hash; } + /**@return The {@link StateDiff} map of the controller*/ public Map getDiff() { return stateDiff.state; } + /** + * Saves the {@link StateDiff} and referencing {@link Hash} identifier to the database. + * + * @param tangle The tangle reference for the database + * @return True if the {@link StateDiff} was saved correctly, False if not + * @throws Exception Thrown if there is an error while saving the {@link StateDiff} + */ public boolean store(Tangle tangle) throws Exception { //return Tangle.instance().save(stateDiff, hash).get(); return tangle.save(stateDiff, hash); } + /** + * Deletes the {@link StateDiff} and referencing {@link Hash} identifier from the database. + * + * @param tangle The tangle reference for the database + * @throws Exception Thrown if there is an error while removing the {@link StateDiff} + */ public void delete(Tangle tangle) throws Exception { tangle.delete(StateDiff.class, hash); } diff --git a/src/main/java/com/iota/iri/controllers/TagViewModel.java b/src/main/java/com/iota/iri/controllers/TagViewModel.java index f3b7c20b8c..17a54c2f4d 100644 --- a/src/main/java/com/iota/iri/controllers/TagViewModel.java +++ b/src/main/java/com/iota/iri/controllers/TagViewModel.java @@ -12,72 +12,130 @@ import java.util.Map; import java.util.Set; +/** + * Acts as a controller interface for a {@link Tag} set. These controllers are used within a + * {@link TransactionViewModel} to manipulate a {@link Tag} set. + */ public class TagViewModel implements HashesViewModel { private Tag self; private Indexable hash; + /** + * Creates an empty Tag set controller. This controller is created using a given hash identifier. + * + * @param hash The hash identifier that the {@link TagViewModel} will be referenced by + */ public TagViewModel(Hash hash) { this.hash = hash; } + /** + * Constructor for a {@link Tag} set controller from an existing {@link Tag} set. If the set is empty, a new + * {@link Tag} set is created. + * + * @param hashes The {@link Tag} set that the controller will be created from + * @param hash The {@link Hash} identifier that acts as a reference for the {@link Tag} set + */ private TagViewModel(Tag hashes, Indexable hash) { self = hashes == null || hashes.set == null ? new Tag(): hashes; this.hash = hash; } + /** + * Creates a new {@link Tag} set controller by converting a {@link com.iota.iri.model.persistables.Hashes} + * referenced by the provided {@link Hash} identifer. This controller is generated by extracting the + * {@link com.iota.iri.model.persistables.Hashes} set from the database using the {@link Hash} identifier + * and casting this set to a {@link Tag} set. This set is then paired with the {@link Hash} identifier to create + * and return a new {@link TagViewModel}. + * + * + * @param tangle The tangle reference for the database to find the {@link Tag} set in + * @param hash The hash identifier for the {@link Tag} set that needs to be found + * @param model The provided {@link Hash} set to be converted + * @return The {@link TagViewModel} controller generated + * @throws Exception Thrown if the database cannot load an {@link Tag} set from the reference {@link Hash} + */ private static TagViewModel load(Tangle tangle, Indexable hash, Class model) throws Exception { return new TagViewModel((Tag) tangle.load(model, hash), hash); } + /** + * Creates a new {@link Tag} set controller. This controller is created by extracting the {@link Tag} set + * from the database using the provided {@link Hash} identifier. + * + * @param tangle The tangle reference for the database to find the {@link Tag} set in + * @param hash The hash identifier for the {@link Tag} set that needs to be found + * @return The {@link TagViewModel} controller generated + * @throws Exception Thrown if the database cannot load an {@link Tag} set from the reference {@link Hash} + */ public static TagViewModel load(Tangle tangle, Indexable hash) throws Exception { return load(tangle, hash, Tag.class); } + /** + * Creates a new {@link ObsoleteTag} set controller. This controller is created by loading the {@link ObsoleteTag} + * set referenced by the {@link Hash} identifier from the database, and loading a new {@link TagViewModel} for it. + * + * @param tangle The tangle reference for the database to find the {@link Tag} set in + * @param hash The hash identifier for the {@link Tag} set that needs to be found + * @return The {@link TagViewModel} controller generated + * @throws Exception Thrown if the database cannot load an {@link Tag} set from the reference {@link Hash} + */ public static TagViewModel loadObsolete(Tangle tangle, Indexable hash) throws Exception { return load(tangle, hash, ObsoleteTag.class); } - public static Map.Entry getEntry(Hash hash, Hash hashToMerge) throws Exception { - Tag hashes = new Tag(); - hashes.set.add(hashToMerge); - return new HashMap.SimpleEntry<>(hash, hashes); + /** + * Fetches the first persistable {@link Tag} set from the database and generates a new + * {@link TagViewModel} from it. If no {@link Tag} sets exist in the database, it will return null. + * + * @param tangle the tangle reference for the database + * @return The new {@link TagViewModel} + * @throws Exception Thrown if the database fails to return a first object + */ + public static TagViewModel first(Tangle tangle) throws Exception { + Pair tagPair = tangle.getFirst(Tag.class, Hash.class); + if(tagPair != null && tagPair.hi != null) { + return new TagViewModel((Tag) tagPair.hi, (Hash) tagPair.low); + } + return null; } + @Override public boolean store(Tangle tangle) throws Exception { return tangle.save(self, hash); } + @Override public int size() { return self.set.size(); } + @Override public boolean addHash(Hash theHash) { return getHashes().add(theHash); } + @Override public Indexable getIndex() { return hash; } + @Override public Set getHashes() { return self.set; } + @Override public void delete(Tangle tangle) throws Exception { tangle.delete(Tag.class,hash); } - public static TagViewModel first(Tangle tangle) throws Exception { - Pair bundlePair = tangle.getFirst(Tag.class, Hash.class); - if(bundlePair != null && bundlePair.hi != null) { - return new TagViewModel((Tag) bundlePair.hi, (Hash) bundlePair.low); - } - return null; - } + @Override public TagViewModel next(Tangle tangle) throws Exception { - Pair bundlePair = tangle.next(Tag.class, hash); - if(bundlePair != null && bundlePair.hi != null) { - return new TagViewModel((Tag) bundlePair.hi, (Hash) bundlePair.low); + Pair tagPair = tangle.next(Tag.class, hash); + if(tagPair != null && tagPair.hi != null) { + return new TagViewModel((Tag) tagPair.hi, (Hash) tagPair.low); } return null; } diff --git a/src/main/java/com/iota/iri/controllers/TipsViewModel.java b/src/main/java/com/iota/iri/controllers/TipsViewModel.java index 30b028a854..05d6bf0874 100644 --- a/src/main/java/com/iota/iri/controllers/TipsViewModel.java +++ b/src/main/java/com/iota/iri/controllers/TipsViewModel.java @@ -8,8 +8,14 @@ import com.iota.iri.model.Hash; +/** + * Acts as a controller interface for a Tips set. A tips set is a a First In First Out cache for + * {@link com.iota.iri.model.persistables.Transaction} objects that have no children. Tips are stored in the + * {@link TipsViewModel} until they are deemed solid or are removed from the cache. + */ public class TipsViewModel { + /** The maximum size of the Tips set*/ public static final int MAX_TIPS = 5000; private final FifoHashCache tips = new FifoHashCache<>(TipsViewModel.MAX_TIPS); @@ -18,12 +24,22 @@ public class TipsViewModel { private final SecureRandom seed = new SecureRandom(); private final Object sync = new Object(); + /** + * Adds a {@link Hash} object to the tip cache in a synchronous fashion. + * + * @param hash The {@link Hash} identifier of the object to be added + */ public void addTipHash(Hash hash) { synchronized (sync) { tips.add(hash); } } + /** + * Removes a {@link Hash} object from the tip cache in a synchronous fashion. + * + * @param hash The {@link Hash} identifier of the object to be removed + */ public void removeTipHash(Hash hash) { synchronized (sync) { if (!tips.remove(hash)) { @@ -32,6 +48,16 @@ public void removeTipHash(Hash hash) { } } + /** + * Removes the referenced {@link Hash} object from the Tips cache and adds it to the SolidTips + * cache. + * + *

    + * A solid tip is a transaction that has been stored in the database, and there are no missing transactions in its history. + *

    + * + * @param tip The {@link Hash} identifier for the object that will be set to solid + */ public void setSolid(Hash tip) { synchronized (sync) { if (tips.remove(tip)) { @@ -40,6 +66,12 @@ public void setSolid(Hash tip) { } } + /** + * Iterates through all solid and non-solid tips and compiles them into one {@link Hash} set to be returned. This + * does so in a synchronised fashion. + * + * @return The {@link Hash} set containing all solid and non-solid tips + */ public Set getTips() { Set hashes = new HashSet<>(); synchronized (sync) { @@ -57,6 +89,13 @@ public Set getTips() { return hashes; } + /** + * Returns a random tip by generating a random integer within the range of the SolidTips set, and iterates + * through the set until a hash is returned. If there are no Solid tips available, then + * getRandomNonSolidTipHash is called and returned instead. + * + * @return A random Solid tip if available, a random NonSolid tip if not + */ public Hash getRandomSolidTipHash() { synchronized (sync) { int size = solidTips.size(); @@ -75,6 +114,12 @@ public Hash getRandomSolidTipHash() { } } + /** + * Returns a random tip by generating integer within the range of the Tips set, and iterates through the + * set until a hash is returned. If there are no tips available, then null is returned instead. + * + * @return A random tip if available, null if not + */ public Hash getRandomNonSolidTipHash() { synchronized (sync) { int size = tips.size(); @@ -93,34 +138,65 @@ public Hash getRandomNonSolidTipHash() { } } + /** + * Fetches the size of the Tips set in a synchronised fashion + * @return The size of the set + */ public int nonSolidSize() { synchronized (sync) { return tips.size(); } } - + + /** + * Fetches the size of the SolidTips set in a synchronised fashion + * @return The size of the set + */ public int solidSize() { synchronized (sync) { return solidTips.size(); } } + /** + * Fetches the size of the Tips set and SolidTipsset combined. This does so in a synchronised + * fashion. + * @return The size of both sets combined + */ public int size() { synchronized (sync) { return tips.size() + solidTips.size(); } } + /** + * A First In First Out hash set for storing Tip transactions. + * + * @param The class of object that will be stored in the hash set + */ private class FifoHashCache { private final int capacity; private final LinkedHashSet set; + /** + * Constructor for a Fifo LinkedHashSet Hash set of a given size. + * + * @param capacity The maximum size allocated for the set + */ public FifoHashCache(int capacity) { this.capacity = capacity; this.set = new LinkedHashSet<>(); } + /** + * Determines if there is vacant space available in the set, and adds the provided object to the set if so. If + * there is no space available, the set removes the next available object iteratively until there is room + * available. + * + * @param key The {@link Hash} identifier for the object that will be added to the set + * @return True if the new objects have been added, False if not + */ public boolean add(K key) { int vacancy = this.capacity - this.set.size(); if (vacancy <= 0) { @@ -133,14 +209,24 @@ public boolean add(K key) { return this.set.add(key); } + /** + * Removes the referenced object from the set. + * + * @param key The {@link Hash} identifier for the object that will be removed from the set + * @return True if the object is removed, False if not + */ public boolean remove(K key) { return this.set.remove(key); } + /**@return The integer size of the stored {@link Hash} set*/ public int size() { return this.set.size(); } + /** + * Creates a new iterator for the object set based on the {@link Hash} class of the set. + * @return The class matched iterator for the stored set*/ public Iterator iterator() { return this.set.iterator(); } diff --git a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java index 6f70c51641..b64177652b 100644 --- a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java +++ b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java @@ -15,15 +15,21 @@ import java.util.*; +/** + * Controller class for {@link Transaction} sets. A {@link TransactionViewModel} stores a {@link HashesViewModel} for + * each component of the {@link Transaction} within it. + */ public class TransactionViewModel { - private final Transaction transaction; + /**Length of a transaction object in trytes*/ public static final int SIZE = 1604; private static final int TAG_SIZE_IN_BYTES = 17; // = ceil(81 TRITS / 5 TRITS_PER_BYTE) + /**Total supply of IOTA available in the network. Used for ensuring a balanced ledger state and bundle balances*/ public static final long SUPPLY = 2779530283277761L; // = (3^33 - 1) / 2 + /**The predefined offset position and size (in trits) for the varying components of a transaction object*/ public static final int SIGNATURE_MESSAGE_FRAGMENT_TRINARY_OFFSET = 0, SIGNATURE_MESSAGE_FRAGMENT_TRINARY_SIZE = 6561; public static final int ADDRESS_TRINARY_OFFSET = SIGNATURE_MESSAGE_FRAGMENT_TRINARY_OFFSET + SIGNATURE_MESSAGE_FRAGMENT_TRINARY_SIZE, ADDRESS_TRINARY_SIZE = 243; public static final int VALUE_TRINARY_OFFSET = ADDRESS_TRINARY_OFFSET + ADDRESS_TRINARY_SIZE, VALUE_TRINARY_SIZE = 81, VALUE_USABLE_TRINARY_SIZE = 33; @@ -46,14 +52,14 @@ public class TransactionViewModel { public static final int ESSENCE_TRINARY_OFFSET = ADDRESS_TRINARY_OFFSET, ESSENCE_TRINARY_SIZE = ADDRESS_TRINARY_SIZE + VALUE_TRINARY_SIZE + OBSOLETE_TAG_TRINARY_SIZE + TIMESTAMP_TRINARY_SIZE + CURRENT_INDEX_TRINARY_SIZE + LAST_INDEX_TRINARY_SIZE; - + /**Stores the {@link HashesViewModel} for the {@link Transaction} components here*/ private AddressViewModel address; private ApproveeViewModel approovers; private TransactionViewModel trunk; private TransactionViewModel branch; private final Hash hash; - + /**Transaction Types*/ public final static int GROUP = 0; // transactions GROUP means that's it's a non-leaf node (leafs store transaction value) public final static int PREFILLED_SLOT = 1; // means that we know only hash of the tx, the rest is unknown yet: only another tx references that hash public final static int FILLED_SLOT = -1; // knows the hash only coz another tx references that hash @@ -61,37 +67,30 @@ public class TransactionViewModel { private byte[] trits; public int weightMagnitude; - public static void fillMetadata(Tangle tangle, TransactionViewModel transactionViewModel) throws Exception { - if (Hash.NULL_HASH.equals(transactionViewModel.getHash())) { - return; - } - if(transactionViewModel.getType() == FILLED_SLOT && !transactionViewModel.transaction.parsed) { - tangle.saveBatch(transactionViewModel.getMetadataSaveBatch()); - } - } - - public static TransactionViewModel find(Tangle tangle, byte[] hash) throws Exception { - TransactionViewModel transactionViewModel = new TransactionViewModel((Transaction) tangle.find(Transaction.class, hash), HashFactory.TRANSACTION.create(hash)); - fillMetadata(tangle, transactionViewModel); - return transactionViewModel; - } - - public static TransactionViewModel fromHash(Tangle tangle, final Hash hash) throws Exception { - TransactionViewModel transactionViewModel = new TransactionViewModel((Transaction) tangle.load(Transaction.class, hash), hash); - fillMetadata(tangle, transactionViewModel); - return transactionViewModel; - } - public static boolean mightExist(Tangle tangle, Hash hash) throws Exception { - return tangle.maybeHas(Transaction.class, hash); - } + /** + * Constructor for a {@link Transaction} set controller interface. This controller is used to interact with and + * manipulate a provided {@link Transaction} set. + * + * @param transaction {@link Transaction} set that the {@link TransactionViewModel} will be created for + * @param hash The {@link Hash} identifier of the {@link Transaction} set + */ public TransactionViewModel(final Transaction transaction, Hash hash) { this.transaction = transaction == null || transaction.bytes == null ? new Transaction(): transaction; this.hash = hash == null? Hash.NULL_HASH: hash; weightMagnitude = this.hash.trailingZeros(); } + /** + * Constructor for a {@link Transaction} set controller interface. A new {@link Transaction} set is created from + * the provided trit array, if the array is of the correct size. If it is not the correct size, a new byte + * array of the correct size is created, the trit array is copied into it, and the bytes are stored in the new + * {@link Transaction} set. This {@link Transaction} set is then indexed by the provided {@link Hash} identifier. + * + * @param trits The input trits that the {@link Transaction} and {@link TransactionViewModel} will be created from. + * @param hash The {@link TransactionHash} identifier of the {@link Transaction} set + */ public TransactionViewModel(final byte[] trits, Hash hash) { transaction = new Transaction(); @@ -113,10 +112,164 @@ public TransactionViewModel(final byte[] trits, Hash hash) { transaction.type = FILLED_SLOT; } + /** + * Creates a new controller for a {@link Transaction} set referenced by a given {@link Hash} identifier. + * The controller will be created and its metadata filled provided the {@link Transaction} object exists in the + * database. + * + * @param tangle The tangle reference for the database + * @param hash The {@link Hash} identifier to search with + * @return The {@link TransactionViewModel} with its Metadata filled in. + * @throws Exception Thrown if there is an error loading the {@link Transaction} object from the database + */ + public static TransactionViewModel fromHash(Tangle tangle, final Hash hash) throws Exception { + TransactionViewModel transactionViewModel = new TransactionViewModel((Transaction) tangle.load(Transaction.class, hash), hash); + fillMetadata(tangle, transactionViewModel); + return transactionViewModel; + } + + /** + * This method checks the {@link com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider} to determine if the + * {@link Transaction} object might exist in the database. If it definitively does not exist, it will return False. + * + * @param tangle The tangle reference for the database + * @param hash The {@link Hash} identifier of the object you are looking for + * @return True if the key might exist in the database, False if it definitively does not + * @throws Exception Thrown if there is an error checking the database + */ + public static boolean mightExist(Tangle tangle, Hash hash) throws Exception { + return tangle.maybeHas(Transaction.class, hash); + } + + /** + * Determines whether the {@link Transaction} object exists in the database or not. + * + * @param tangle The tangle reference for the database. + * @param hash The {@link Hash} identifier for the {@link Transaction} object + * @return True if the transaction exists in the database, False if not + * @throws Exception Thrown if there is an error determining if the transaction exists or not + */ + public static boolean exists(Tangle tangle, Hash hash) throws Exception { + return tangle.exists(Transaction.class, hash); + } + + /** + * Returns the total number of {@link Transaction} objects stored in a database. + * + * @param tangle The tangle reference for the database. + * @return The integer count of total {@link Transaction} objects in the database + * @throws Exception Thrown if there is an error getting the count of objects + */ public static int getNumberOfStoredTransactions(Tangle tangle) throws Exception { return tangle.getCount(Transaction.class).intValue(); } + /** + * Converts the given byte array to the a new trit array of length {@value TRINARY_SIZE}. + * @param transactionBytes The byte array to be converted to trits + * @return The trit conversion of the byte array + */ + public static byte[] trits(byte[] transactionBytes) { + byte[] trits; + trits = new byte[TRINARY_SIZE]; + if(transactionBytes != null) { + Converter.getTrits(transactionBytes, trits); + } + return trits; + } + + public static Set getMissingTransactions(Tangle tangle) throws Exception { + return tangle.keysWithMissingReferences(Approvee.class, Transaction.class); + } + + /** + * Creates a new controller using a byte array that will be converted to a {@link Hash} identifier. The new + * {@link TransactionViewModel} is created for the {@link Transaction} object referenced by this {@link Hash} + * identifier, provided the {@link Transaction} exists in the database. + * + * @param tangle The tangle reference for the database + * @param hash The source that the {@link Hash} identifier will be created from, and the {@link Transaction} object + * will be fetched from the database from + * @return The {@link TransactionViewModel} with its Metadata filled in. + * @throws Exception Thrown if the database fails to find the {@link Transaction} object + */ + public static TransactionViewModel find(Tangle tangle, byte[] hash) throws Exception { + TransactionViewModel transactionViewModel = new TransactionViewModel((Transaction) tangle.find(Transaction.class, hash), HashFactory.TRANSACTION.create(hash)); + fillMetadata(tangle, transactionViewModel); + return transactionViewModel; + } + + /** + * Fetches the first persistable {@link Transaction} object from the database and generates a new + * {@link TransactionViewModel} from it. If no objects exist in the database, it will return null. + * + * @param tangle the tangle reference for the database. + * @return The new {@link TransactionViewModel}. + * @throws Exception Thrown if the database fails to return a first object. + */ + public static TransactionViewModel first(Tangle tangle) throws Exception { + Pair transactionPair = tangle.getFirst(Transaction.class, Hash.class); + if(transactionPair != null && transactionPair.hi != null) { + return new TransactionViewModel((Transaction) transactionPair.hi, (Hash) transactionPair.low); + } + return null; + } + + /** + * Updates a set of {@link Transaction} objects to solid. Used by the {@link com.iota.iri.TransactionValidator} to + * solidify groups of analyzed {@link Transaction} sets. + * + * @param tangle The tangle reference for the database. + * @param analyzedHashes A list of {@link TransactionHash} identifiers to be checked for solidity. + * @throws Exception Thrown if there is a failure to generate a controller, or if there is a failure to update + */ + public static void updateSolidTransactions(Tangle tangle, final Set analyzedHashes) throws Exception { + Iterator hashIterator = analyzedHashes.iterator(); + TransactionViewModel transactionViewModel; + while(hashIterator.hasNext()) { + transactionViewModel = TransactionViewModel.fromHash(tangle, hashIterator.next()); + + transactionViewModel.updateHeights(tangle); + + if(!transactionViewModel.isSolid()) { + transactionViewModel.updateSolid(true); + transactionViewModel.update(tangle, "solid|height"); + } + } + } + + /** + * Populates the meta data of the {@link TransactionViewModel}. If the controller {@link Hash} identifier is + * null, it will return with no response. If the {@link Transaction} object has not been parsed, and the + * {@link TransactionViewModel} type is FILLED_SLOT, the saved metadata batch will be saved to the + * database. + * + * @param tangle The tangle reference for the database. + * @param transactionViewModel The {@link TransactionViewModel} whose Metadata is to be filled. + * @throws Exception Thrown if the database fails to save the batch of data. + */ + + public static void fillMetadata(Tangle tangle, TransactionViewModel transactionViewModel) throws Exception { + if (Hash.NULL_HASH.equals(transactionViewModel.getHash())) { + return; + } + if(transactionViewModel.getType() == FILLED_SLOT && !transactionViewModel.transaction.parsed) { + tangle.saveBatch(transactionViewModel.getMetadataSaveBatch()); + } + } + + /** + * This method updates the metadata contained in the {@link Transaction} object, and updates the object in the + * database. First, all the most recent {@link Hash} identifiers are fetched to make sure the object's metadata is + * up to date. Then it checks if the current {@link TransactionHash} is null. If it is, then the method immediately + * returns false, and if not, it attempts to update the {@link Transaction} object and the referencing {@link Hash} + * identifier in the database. + * + * @param tangle The tangle reference for the database + * @param item The string identifying the purpose of the update + * @return True if the update was successful, False if it failed + * @throws Exception Thrown if any of the metadata fails to fetch, or if the database update fails + */ public boolean update(Tangle tangle, String item) throws Exception { getAddressHash(); getTrunkTransactionHash(); @@ -132,6 +285,15 @@ public boolean update(Tangle tangle, String item) throws Exception { return tangle.update(transaction, hash, item); } + /** + * Retrieves the {@link TransactionViewModel} for the branch {@link Transaction} object referenced by this + * {@link TransactionViewModel}. If the controller doesn't already exist, a new one is created from the branch + * transaction {@link Hash} identifier. + * + * @param tangle The tangle reference for the database. + * @return The branch transaction {@link TransactionViewModel} + * @throws Exception Thrown if no branch is found when creating the branch {@link TransactionViewModel} + */ public TransactionViewModel getBranchTransaction(Tangle tangle) throws Exception { if(branch == null) { branch = TransactionViewModel.fromHash(tangle, getBranchTransactionHash()); @@ -139,6 +301,15 @@ public TransactionViewModel getBranchTransaction(Tangle tangle) throws Exception return branch; } + /** + * Retrieves the {@link TransactionViewModel} for the trunk {@link Transaction} object referenced by this + * {@link TransactionViewModel}. If the controller doesn't already exist, a new one is created from the trunk + * transaction {@link Hash} identifier. + * + * @param tangle The tangle reference for the database. + * @return The trunk transaction {@link TransactionViewModel} + * @throws Exception Thrown if no trunk is found when creating the trunk {@link TransactionViewModel} + */ public TransactionViewModel getTrunkTransaction(Tangle tangle) throws Exception { if(trunk == null) { trunk = TransactionViewModel.fromHash(tangle, getTrunkTransactionHash()); @@ -146,23 +317,31 @@ public TransactionViewModel getTrunkTransaction(Tangle tangle) throws Exception return trunk; } - public static byte[] trits(byte[] transactionBytes) { - byte[] trits; - trits = new byte[TRINARY_SIZE]; - if(transactionBytes != null) { - Converter.getTrits(transactionBytes, trits); - } - return trits; - } - + /** + * @return The trits stored in the {@link TransactionViewModel} . If the trits aren't available in the controller, + * the trits are generated from the transaction object and stored in the controller {@link #trits()} variable. + */ public synchronized byte[] trits() { return (trits == null) ? (trits = trits(transaction.bytes)) : trits; } + /** + * Deletes the {@link Transaction} object from the database + * @param tangle The tangle reference for the database + * @throws Exception Thrown if there is an error removing the object + */ public void delete(Tangle tangle) throws Exception { tangle.delete(Transaction.class, hash); } + /** + * Stores the {@link Transaction} object to the tangle, including the metadata and indexing based on {@link Bundle}, + * {@link Address}, {@link Tag}, {@link #trunk} and {@link #branch}. + * + * @return The list of {@link Hash} objects indexed by the {@link TransactionHash} identifier. Returns False if + * there is a problem populating the list. + */ + public List> getMetadataSaveBatch() throws Exception { List> hashesList = new ArrayList<>(); hashesList.add(new Pair<>(getAddressHash(), new Address(hash))); @@ -176,6 +355,14 @@ public List> getMetadataSaveBatch() throws Exceptio return hashesList; } + /** + * Fetches a list of all {@link Transaction} component and {@link Hash} identifier pairs from the stored metadata. + * The method then ensures that the {@link Transaction#bytes} are present before adding the {@link Transaction} and + * {@link Hash} identifier to the already compiled list of {@link Transaction} components. + * + * @return A complete list of all {@link Transaction} component objects paired with their {@link Hash} identifiers + * @throws Exception Thrown if the metadata fails to fetch, or if the bytes are not retrieved correctly + */ public List> getSaveBatch() throws Exception { List> hashesList = new ArrayList<>(); hashesList.addAll(getMetadataSaveBatch()); @@ -184,15 +371,14 @@ public List> getSaveBatch() throws Exception { return hashesList; } - - public static TransactionViewModel first(Tangle tangle) throws Exception { - Pair transactionPair = tangle.getFirst(Transaction.class, Hash.class); - if(transactionPair != null && transactionPair.hi != null) { - return new TransactionViewModel((Transaction) transactionPair.hi, (Hash) transactionPair.low); - } - return null; - } - + /** + * Fetches the next indexed persistable {@link Transaction} object from the database and generates a new + * {@link TransactionViewModel} from it. If no objects exist in the database, it will return null. + * + * @param tangle the tangle reference for the database. + * @return The new {@link TransactionViewModel}. + * @throws Exception Thrown if the database fails to return a next object. + */ public TransactionViewModel next(Tangle tangle) throws Exception { Pair transactionPair = tangle.next(Transaction.class, hash); if(transactionPair != null && transactionPair.hi != null) { @@ -201,6 +387,16 @@ public TransactionViewModel next(Tangle tangle) throws Exception { return null; } + /** + * This method fetches the saved batch of metadata and orders them into a list of {@link Hash} objects and + * {@link Hash} identifier pairs. If the {@link Hash} identifier of the {@link Transaction} is null, or the database + * already contains the {@link Transaction}, then the method returns False. Otherwise, the method tries to store the + * {@link Transaction} batch into the database. + * + * @param tangle The tangle reference for the database. + * @return True if the {@link Transaction} is stored, False if not. + * @throws Exception Thrown if there is an error fetching the batch or storing in the database. + */ public boolean store(Tangle tangle) throws Exception { if (hash.equals(Hash.NULL_HASH) || exists(tangle, hash)) { return false; @@ -213,6 +409,17 @@ public boolean store(Tangle tangle) throws Exception { return tangle.saveBatch(batch); } + /** + * Gets the {@link ApproveeViewModel} of a {@link Transaction}. If the current {@link ApproveeViewModel} is null, + * a new one is created using the transaction {@link Hash} identifier. + * + * An {@link Approvee} is a transaction in the tangle that references, and therefore approves, this transaction + * directly. + * + * @param tangle The tangle reference for the database + * @return The {@link ApproveeViewModel} + * @throws Exception Thrown if there is a failure to create a controller from the transaction hash + */ public ApproveeViewModel getApprovers(Tangle tangle) throws Exception { if(approovers == null) { approovers = ApproveeViewModel.load(tangle, hash); @@ -220,18 +427,40 @@ public ApproveeViewModel getApprovers(Tangle tangle) throws Exception { return approovers; } + /** + * Gets the {@link Transaction#type}. The type can be one of 3: + *
      + *
    • PREFILLED_SLOT: 1
      • + *
    • FILLED_SLOT: -1
      • + *
    • GROUP: 0
      • + *
    + * + * @return The current type of the transaction. + */ public final int getType() { return transaction.type; } + /** + * Sets the {@link Transaction#arrivalTime}. + * @param time The time to be set in the {@link Transaction} + */ public void setArrivalTime(long time) { transaction.arrivalTime = time; } + /**@return The {@link Transaction#arrivalTime}*/ public long getArrivalTime() { return transaction.arrivalTime; } + /** + * Gets the stored {@link Transaction#bytes}. If the {@link Transaction#bytes} are null, a new byte + * array is created and stored from the {@link #trits}. If the {@link #trits} are also null, then a null byte array + * is returned. + * + * @return The stored {@link Transaction#bytes} array + */ public byte[] getBytes() { if(transaction.bytes == null || transaction.bytes.length != SIZE) { transaction.bytes = new byte[SIZE]; @@ -242,10 +471,18 @@ public byte[] getBytes() { return transaction.bytes; } + /**@return The transaction {@link Hash} identifier*/ public Hash getHash() { return hash; } + /** + * Gets the {@link AddressViewModel} associated with this {@link Transaction}. + * + * @param tangle The tangle reference for the database. + * @return The {@link AddressViewModel} of the {@link Transaction}. + * @throws Exception If the address cannot be found in the database, an exception is thrown. + */ public AddressViewModel getAddress(Tangle tangle) throws Exception { if(address == null) { address = AddressViewModel.load(tangle, getAddressHash()); @@ -253,10 +490,22 @@ public AddressViewModel getAddress(Tangle tangle) throws Exception { return address; } + /** + * Gets the {@link TagViewModel} associated with this {@link Transaction}. + * + * @param tangle The tangle reference for the database. + * @return The {@link TagViewModel} of the {@link Transaction}. + * @throws Exception If the address cannot be found in the database, an exception is thrown. + */ public TagViewModel getTag(Tangle tangle) throws Exception { return TagViewModel.load(tangle, getTagValue()); } + /** + * Gets the {@link AddressHash} identifier of a {@link Transaction}. + * + * @return The {@link AddressHash} identifier. + */ public Hash getAddressHash() { if(transaction.address == null) { transaction.address = HashFactory.ADDRESS.create(trits(), ADDRESS_TRINARY_OFFSET); @@ -264,6 +513,11 @@ public Hash getAddressHash() { return transaction.address; } + /** + * Gets the {@link ObsoleteTagHash} identifier of a {@link Transaction}. + * + * @return The {@link ObsoleteTagHash} identifier. + */ public Hash getObsoleteTagValue() { if(transaction.obsoleteTag == null) { byte[] tagBytes = Converter.allocateBytesForTrits(OBSOLETE_TAG_TRINARY_SIZE); @@ -274,6 +528,11 @@ public Hash getObsoleteTagValue() { return transaction.obsoleteTag; } + /** + * Gets the {@link BundleHash} identifier of a {@link Transaction}. + * + * @return The {@link BundleHash} identifier. + */ public Hash getBundleHash() { if(transaction.bundle == null) { transaction.bundle = HashFactory.BUNDLE.create(trits(), BUNDLE_TRINARY_OFFSET); @@ -281,6 +540,11 @@ public Hash getBundleHash() { return transaction.bundle; } + /** + * Gets the trunk {@link TransactionHash} identifier of a {@link Transaction}. + * + * @return The trunk {@link TransactionHash} identifier. + */ public Hash getTrunkTransactionHash() { if(transaction.trunk == null) { transaction.trunk = HashFactory.TRANSACTION.create(trits(), TRUNK_TRANSACTION_TRINARY_OFFSET); @@ -288,6 +552,11 @@ public Hash getTrunkTransactionHash() { return transaction.trunk; } + /** + * Gets the branch {@link TransactionHash} identifier of a {@link Transaction}. + * + * @return The branch {@link TransactionHash} identifier. + */ public Hash getBranchTransactionHash() { if(transaction.branch == null) { transaction.branch = HashFactory.TRANSACTION.create(trits(), BRANCH_TRANSACTION_TRINARY_OFFSET); @@ -295,6 +564,11 @@ public Hash getBranchTransactionHash() { return transaction.branch; } + /** + * Gets the {@link TagHash} identifier of a {@link Transaction}. + * + * @return The {@link TagHash} identifier. + */ public Hash getTagValue() { if(transaction.tag == null) { byte[] tagBytes = Converter.allocateBytesForTrits(TAG_TRINARY_SIZE); @@ -304,19 +578,48 @@ public Hash getTagValue() { return transaction.tag; } + /** + * Gets the {@link Transaction#attachmentTimestamp}. The Attachment Timestapm is used to show when + * a transaction has been attached to the database. + * + * @return The {@link Transaction#attachmentTimestamp} + */ public long getAttachmentTimestamp() { return transaction.attachmentTimestamp; } + + /** + * Gets the {@link Transaction#attachmentTimestampLowerBound}. The Attachment Timestamp Lower Bound + * is the earliest timestamp a transaction can have. + * + * @return The {@link Transaction#attachmentTimestampLowerBound} + */ public long getAttachmentTimestampLowerBound() { return transaction.attachmentTimestampLowerBound; } + + /** + * Gets the {@link Transaction#attachmentTimestampUpperBound}. The Attachment Timestamp Upper Bound + * is the maximum timestamp a transaction can have. + * + * @return The {@link Transaction#attachmentTimestampUpperBound} + */ public long getAttachmentTimestampUpperBound() { return transaction.attachmentTimestampUpperBound; } - + /**@return The {@link Transaction#value}*/ public long value() { return transaction.value; } + /** + * Updates the {@link Transaction#validity} in the database. + * + * The validity can be one of three states: + * 1: Valid; -1: Invalid; 0: Unknown + * @param tangle The tangle reference for the database + * @param validity The state of validity that the {@link Transaction} will be updated to + * @throws Exception Thrown if there is an error with the update + */ public void setValidity(Tangle tangle, int validity) throws Exception { if(transaction.validity != validity) { transaction.validity = validity; @@ -324,32 +627,52 @@ public void setValidity(Tangle tangle, int validity) throws Exception { } } + /**@return The current stored {@link Transaction#validity}*/ public int getValidity() { return transaction.validity; } + /**@return The {@link Transaction#currentIndex} in its bundle*/ public long getCurrentIndex() { return transaction.currentIndex; } + /** + * Creates an array copy of the signature message fragment of the {@link Transaction} and returns it. + * @return The signature message fragment in array format. + */ public byte[] getSignature() { return Arrays.copyOfRange(trits(), SIGNATURE_MESSAGE_FRAGMENT_TRINARY_OFFSET, SIGNATURE_MESSAGE_FRAGMENT_TRINARY_SIZE); } + /**@return The stored {@link Transaction#timestamp}*/ public long getTimestamp() { return transaction.timestamp; } + /** + * Creates a byte array of the size {@value NONCE_TRINARY_SIZE} and copies the nonce section of the transaction + * trits into the array. This array is then returned. + * + * @return A byte array containing the nonce of the transaction + */ public byte[] getNonce() { byte[] nonce = Converter.allocateBytesForTrits(NONCE_TRINARY_SIZE); Converter.bytes(trits(), NONCE_TRINARY_OFFSET, nonce, 0, trits().length); return nonce; } + /**@return The {@link Transaction#lastIndex} of the transaction bundle*/ public long lastIndex() { return transaction.lastIndex; } + /** + * Fetches the {@link Transaction#tag}, and converts the transaction trits for the + * {@link Transaction#attachmentTimestamp}, the {@link Transaction#attachmentTimestampLowerBound}, and the + * {@link Transaction#attachmentTimestampUpperBound} to long values.The method then sets these values to the + * {@link TransactionViewModel} metadata. + */ public void setAttachmentData() { getTagValue(); transaction.attachmentTimestamp = Converter.longValue(trits(), ATTACHMENT_TIMESTAMP_TRINARY_OFFSET, ATTACHMENT_TIMESTAMP_TRINARY_SIZE); @@ -357,6 +680,13 @@ public void setAttachmentData() { transaction.attachmentTimestampUpperBound = Converter.longValue(trits(), ATTACHMENT_TIMESTAMP_UPPER_BOUND_TRINARY_OFFSET, ATTACHMENT_TIMESTAMP_UPPER_BOUND_TRINARY_SIZE); } + + /** + * Converts the {@link Transaction#value}, {@link Transaction#timestamp}, {@link Transaction#currentIndex} and + * {@link Transaction#lastIndex} from trits to long values and assigns them to the {@link TransactionViewModel} + * metadata. The method then determines if the {@link Transaction#bytes} are null or not. If so the + * {@link Transaction#type} is set to {@link #PREFILLED_SLOT}, and if not it is set to {@link #FILLED_SLOT}. + */ public void setMetadata() { transaction.value = Converter.longValue(trits(), VALUE_TRINARY_OFFSET, VALUE_USABLE_TRINARY_SIZE); transaction.timestamp = Converter.longValue(trits(), TIMESTAMP_TRINARY_OFFSET, TIMESTAMP_TRINARY_SIZE); @@ -366,29 +696,15 @@ public void setMetadata() { transaction.type = transaction.bytes == null ? TransactionViewModel.PREFILLED_SLOT : TransactionViewModel.FILLED_SLOT; } - public static boolean exists(Tangle tangle, Hash hash) throws Exception { - return tangle.exists(Transaction.class, hash); - } - - public static Set getMissingTransactions(Tangle tangle) throws Exception { - return tangle.keysWithMissingReferences(Approvee.class, Transaction.class); - } - - public static void updateSolidTransactions(Tangle tangle, final Set analyzedHashes) throws Exception { - Iterator hashIterator = analyzedHashes.iterator(); - TransactionViewModel transactionViewModel; - while(hashIterator.hasNext()) { - transactionViewModel = TransactionViewModel.fromHash(tangle, hashIterator.next()); - - transactionViewModel.updateHeights(tangle); - - if(!transactionViewModel.isSolid()) { - transactionViewModel.updateSolid(true); - transactionViewModel.update(tangle, "solid|height"); - } - } - } + /** + * Updates the {@link Transaction#solid} value of the referenced {@link Transaction} object. + * + * Used by the {@link com.iota.iri.TransactionValidator} to quickly set the solidity of a {@link Transaction} set. + * + * @param solid The solidity of the transaction in the database + * @return True if the {@link Transaction#solid} has been updated, False if not. + */ public boolean updateSolid(boolean solid) throws Exception { if(solid != transaction.solid) { transaction.solid = solid; @@ -397,14 +713,25 @@ public boolean updateSolid(boolean solid) throws Exception { return false; } + /**@return True if {@link Transaction#solid} is True (exists in the database), False if not*/ public boolean isSolid() { return transaction.solid; } + /**@return The {@link Transaction#snapshot} index*/ public int snapshotIndex() { return transaction.snapshot; } + /** + * Sets the current {@link Transaction#snapshot} index. + * + * This is used to set a milestone transactions index. + * + * @param tangle The tangle reference for the database. + * @param index The new index to be attached to the {@link Transaction} object + * @throws Exception Thrown if the database update does not return correctly + */ public void setSnapshot(Tangle tangle, final int index) throws Exception { if ( index != transaction.snapshot ) { transaction.snapshot = index; @@ -413,14 +740,15 @@ public void setSnapshot(Tangle tangle, final int index) throws Exception { } /** - * This method is the setter for the milestone flag of a transaction. + * This method sets the {@link Transaction#milestone} flag. * - * It gets automatically called by the "Latest Milestone Tracker" and marks transactions that represent a milestone - * accordingly. It first checks if the value has actually changed and then issues a database update. + * It gets automatically called by the {@link com.iota.iri.MilestoneTracker} and marks transactions that represent + * a milestone accordingly. It first checks if the {@link Transaction#milestone} flag has changed and if so, it + * issues a database update. * * @param tangle Tangle instance which acts as a database interface - * @param isMilestone true if the transaction is a milestone and false otherwise - * @throws Exception if something goes wrong while saving the changes to the database + * @param isMilestone True if the {@link Transaction} is a milestone and False if not + * @throws Exception Thrown if there is an error while saving the changes to the database */ public void isMilestone(Tangle tangle, final boolean isMilestone) throws Exception { if (isMilestone != transaction.milestone) { @@ -430,27 +758,38 @@ public void isMilestone(Tangle tangle, final boolean isMilestone) throws Excepti } /** - * This method is the getter for the milestone flag of a transaction. + * This method gets the {@link Transaction#milestone}. * - * The milestone flag indicates if the transaction is a coordinator issued milestone. It allows us to differentiate - * the two types of transactions (normal transactions / milestones) very fast and efficiently without issuing - * further database queries or even full verifications of the signature. If it is set to true one can for example - * use the snapshotIndex() method to retrieve the corresponding MilestoneViewModel object. + * The {@link Transaction#milestone} flag indicates if the {@link Transaction} is a coordinator issued milestone. It + * allows us to differentiate the two types of transactions (normal transactions / milestones) very fast and + * efficiently without issuing further database queries or even full verifications of the signature. If it is set to + * true one can for example use the snapshotIndex() method to retrieve the corresponding + * {@link MilestoneViewModel} object. * - * @return true if the transaction is a milestone and false otherwise + * @return true if the {@link Transaction} is a milestone and false otherwise */ public boolean isMilestone() { return transaction.milestone; } + /**@return The current {@link Transaction#height}*/ public long getHeight() { return transaction.height; } + /** + * Updates the {@link Transaction#height}. + * @param height The new height of the {@link Transaction} + */ private void updateHeight(long height) throws Exception { transaction.height = height; } + /** + * + * @param tangle + * @throws Exception + */ public void updateHeights(Tangle tangle) throws Exception { TransactionViewModel transactionVM = this, trunk = this.getTrunkTransaction(tangle); Stack transactionViewModels = new Stack<>(); @@ -482,9 +821,15 @@ public void updateHeights(Tangle tangle) throws Exception { } } + /** + * Updates the {@link Transaction#sender}. + * @param sender The sender of the {@link Transaction} + */ public void updateSender(String sender) throws Exception { transaction.sender = sender; } + + /**@return The {@link Transaction#sender}*/ public String getSender() { return transaction.sender; } From f02d787d47eb9a04e764c15562a281ea8d7d92c1 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 9 Dec 2018 10:13:13 +0200 Subject: [PATCH 04/63] Update bug_report.md --- .github/ISSUE_TEMPLATE/bug_report.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index f0b91acf98..51bfb2a02c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -12,6 +12,9 @@ If you have general questions on IOTA you can go to https://iota.stackexchange.c ### Bug description A general description of the bug. +### IRI version +What version are you running? + ### Hardware Spec On what hardware is the node running on? From d145e3f07080d60e3fdb9603a22ddb78fcdabd86 Mon Sep 17 00:00:00 2001 From: Hans Moog Date: Wed, 19 Dec 2018 04:30:05 +0100 Subject: [PATCH 05/63] Feat: intermediary commit --- .../spentaddresses/SpentAddressesProvider.java | 13 +++++++++++++ .../spentaddresses/SpentAddressesService.java | 7 +++++++ .../impl/SpentAddressesProviderImpl.java | 6 ++++++ .../impl/SpentAddressesServiceImpl.java | 6 ++++++ 4 files changed, 32 insertions(+) create mode 100644 src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java create mode 100644 src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java create mode 100644 src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java create mode 100644 src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java new file mode 100644 index 0000000000..dd33a27c08 --- /dev/null +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java @@ -0,0 +1,13 @@ +package com.iota.iri.service.spentaddresses; + +import com.iota.iri.model.Hash; + +public interface SpentAddressesProvider { + void containsAddress(Hash addressHash); + + void addAddress(Hash addressHash); + + void shutdown(); + + void writeSpentAddressesToDisk(String basePath); +} diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java new file mode 100644 index 0000000000..f46b342474 --- /dev/null +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java @@ -0,0 +1,7 @@ +package com.iota.iri.service.spentaddresses; + +import com.iota.iri.model.Hash; + +public interface SpentAddressesService { + boolean wasAddressSpent(Hash addressHash); +} diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java new file mode 100644 index 0000000000..1e3dfd29ac --- /dev/null +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -0,0 +1,6 @@ +package com.iota.iri.service.spentaddresses.impl; + +import com.iota.iri.service.spentaddresses.SpentAddressesProvider; + +public class SpentAddressesProviderImpl implements SpentAddressesProvider { +} diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java new file mode 100644 index 0000000000..e3e9a972dd --- /dev/null +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -0,0 +1,6 @@ +package com.iota.iri.service.spentaddresses.impl; + +import com.iota.iri.service.spentaddresses.SpentAddressesService; + +public class SpentAddressesServiceImpl implements SpentAddressesService { +} From 6aa275b0413f4a8cca311ab02449945fd44e450f Mon Sep 17 00:00:00 2001 From: Hans Moog Date: Fri, 21 Dec 2018 10:10:14 +0100 Subject: [PATCH 06/63] Feat: added SpentAddressesProvider --- src/main/java/com/iota/iri/Iota.java | 5 ++ .../iri/model/persistables/SpentAddress.java | 31 +++++++ src/main/java/com/iota/iri/service/API.java | 42 +--------- .../SpentAddressesException.java | 38 +++++++++ .../SpentAddressesProvider.java | 6 +- .../spentaddresses/SpentAddressesService.java | 7 -- .../impl/SpentAddressesProviderImpl.java | 84 +++++++++++++++++++ .../impl/SpentAddressesServiceImpl.java | 6 -- .../rocksDB/RocksDBPersistenceProvider.java | 12 ++- 9 files changed, 166 insertions(+), 65 deletions(-) create mode 100644 src/main/java/com/iota/iri/model/persistables/SpentAddress.java create mode 100644 src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesException.java delete mode 100644 src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java delete mode 100644 src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index e9535e6c91..0a4063a06f 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -20,6 +20,7 @@ import com.iota.iri.service.snapshot.impl.LocalSnapshotManagerImpl; import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; import com.iota.iri.service.snapshot.impl.SnapshotServiceImpl; +import com.iota.iri.service.spentaddresses.impl.SpentAddressesProviderImpl; import com.iota.iri.service.tipselection.EntryPointSelector; import com.iota.iri.service.tipselection.RatingCalculator; import com.iota.iri.service.tipselection.TailFinder; @@ -81,6 +82,8 @@ public class Iota { private static final Logger log = LoggerFactory.getLogger(Iota.class); + public final SpentAddressesProviderImpl spentAddressesProvider; + public final SnapshotProviderImpl snapshotProvider; public final SnapshotServiceImpl snapshotService; @@ -127,6 +130,7 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh this.configuration = configuration; // new refactored instances + spentAddressesProvider = new SpentAddressesProviderImpl(); snapshotProvider = new SnapshotProviderImpl(); snapshotService = new SnapshotServiceImpl(); localSnapshotManager = configuration.getLocalSnapshotsEnabled() @@ -202,6 +206,7 @@ public void init() throws Exception { } private void injectDependencies() throws SnapshotException, TransactionPruningException { + spentAddressesProvider.init(tangle, configuration); snapshotProvider.init(configuration); snapshotService.init(tangle, snapshotProvider, configuration); if (localSnapshotManager != null) { diff --git a/src/main/java/com/iota/iri/model/persistables/SpentAddress.java b/src/main/java/com/iota/iri/model/persistables/SpentAddress.java new file mode 100644 index 0000000000..99c6c8d271 --- /dev/null +++ b/src/main/java/com/iota/iri/model/persistables/SpentAddress.java @@ -0,0 +1,31 @@ +package com.iota.iri.model.persistables; + +import com.iota.iri.storage.Persistable; + +public class SpentAddress implements Persistable { + public boolean exists = false; + + @Override + public byte[] bytes() { + return new byte[0]; + } + + @Override + public void read(byte[] bytes) { + exists = bytes != null; + } + + @Override + public byte[] metadata() { + return new byte[0]; + } + + @Override + public void readMetadata(byte[] bytes) { + } + + @Override + public boolean merge() { + return false; + } +} diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index fd555f1710..f4ffccd618 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -111,8 +111,6 @@ public class API { private final static String overMaxErrorMessage = "Could not complete request"; private final static String invalidParams = "Invalid parameters"; - private ConcurrentHashMap previousEpochsSpentAddresses; - private final static char ZERO_LENGTH_ALLOWED = 'Y'; private final static char ZERO_LENGTH_NOT_ALLOWED = 'N'; private Iota instance; @@ -136,8 +134,6 @@ public API(Iota instance, IXI ixi) { maxBodyLength = configuration.getMaxBodyLength(); testNet = configuration.isTestnet(); - previousEpochsSpentAddresses = new ConcurrentHashMap<>(); - features = Feature.calculateFeatureNames(instance.configuration); } @@ -165,13 +161,8 @@ public API(Iota instance, IXI ixi) { * Starts the server, opening it for HTTP API requests * *
- * - * @throws IOException If we are not on the testnet, and the previousEpochsSpentAddresses files cannot be found. - * Currently this exception is caught in {@link #readPreviousEpochsSpentAddresses(boolean)} */ public void init() throws IOException { - readPreviousEpochsSpentAddresses(testNet); - APIConfig configuration = instance.configuration; final int apiPort = configuration.getPort(); final String apiHost = configuration.getApiHost(); @@ -206,37 +197,6 @@ public void handleRequest(final HttpServerExchange exchange) throws Exception { server.start(); } - /** - * Read the spend addresses from the previous epoch. Used in {@link #wasAddressSpentFrom(Hash)}. - * If this fails, a log is printed. The API will continue to initialize. - * - * @param isTestnet If this node is running on the testnet. If this is true, nothing is loaded. - * @throws IOException If we are not on the testnet and previousEpochsSpentAddresses files cannot be found. - * Currently this exception is caught in {@link #readPreviousEpochsSpentAddresses(boolean)} - */ - private void readPreviousEpochsSpentAddresses(boolean isTestnet) throws IOException { - if (isTestnet) { - return; - } - - String[] previousEpochsSpentAddressesFiles = instance - .configuration - .getPreviousEpochSpentAddressesFiles() - .split(" "); - - for (String previousEpochsSpentAddressesFile : previousEpochsSpentAddressesFiles) { - InputStream in = Snapshot.class.getResourceAsStream(previousEpochsSpentAddressesFile); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { - String line; - while ((line = reader.readLine()) != null) { - this.previousEpochsSpentAddresses.put(HashFactory.ADDRESS.create(line), true); - } - } catch (Exception e) { - log.error("Failed to load resource: {}.", previousEpochsSpentAddressesFile, e); - } - } - } - /** * Sends the API response back as JSON to the requester. * Status code of the HTTP request is also set according to the type of response. @@ -551,7 +511,7 @@ private AbstractResponse wereAddressesSpentFromStatement(List addresses) * @throws Exception When a model could not be loaded. */ private boolean wasAddressSpentFrom(Hash address) throws Exception { - if (previousEpochsSpentAddresses.containsKey(address)) { + if (instance.spentAddressesProvider.containsAddress(address)) { return true; } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesException.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesException.java new file mode 100644 index 0000000000..291aa09224 --- /dev/null +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesException.java @@ -0,0 +1,38 @@ +package com.iota.iri.service.spentaddresses; + +/** + * This class is used to wrap exceptions that are specific to the spent addresses logic. + * + * It allows us to distinct between the different kinds of errors that can happen during the execution of the code. + */ +public class SpentAddressesException extends Exception { + /** + * Constructor of the exception which allows us to provide a specific error message and the cause of the error. + * + * @param message reason why this error occurred + * @param cause wrapped exception that caused this error + */ + public SpentAddressesException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor of the exception which allows us to provide a specific error message without having an underlying + * cause. + * + * @param message reason why this error occurred + */ + public SpentAddressesException(String message) { + super(message); + } + + /** + * Constructor of the exception which allows us to wrap the underlying cause of the error without providing a + * specific reason. + * + * @param cause wrapped exception that caused this error + */ + public SpentAddressesException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java index dd33a27c08..d33d41c785 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java @@ -3,11 +3,9 @@ import com.iota.iri.model.Hash; public interface SpentAddressesProvider { - void containsAddress(Hash addressHash); + boolean containsAddress(Hash addressHash) throws SpentAddressesException; - void addAddress(Hash addressHash); - - void shutdown(); + void addAddress(Hash addressHash) throws SpentAddressesException; void writeSpentAddressesToDisk(String basePath); } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java deleted file mode 100644 index f46b342474..0000000000 --- a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.iota.iri.service.spentaddresses; - -import com.iota.iri.model.Hash; - -public interface SpentAddressesService { - boolean wasAddressSpent(Hash addressHash); -} diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 1e3dfd29ac..f815fbde75 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -1,6 +1,90 @@ package com.iota.iri.service.spentaddresses.impl; +import com.iota.iri.conf.IotaConfig; +import com.iota.iri.model.Hash; +import com.iota.iri.model.HashFactory; +import com.iota.iri.model.persistables.SpentAddress; +import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.spentaddresses.SpentAddressesProvider; +import com.iota.iri.storage.Tangle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; public class SpentAddressesProviderImpl implements SpentAddressesProvider { + private static final Logger log = LoggerFactory.getLogger(SpentAddressesProviderImpl.class); + + private Tangle tangle; + + private IotaConfig config; + + public SpentAddressesProviderImpl() {} + + public SpentAddressesProviderImpl init(Tangle tangle, IotaConfig config) { + this.tangle = tangle; + this.config = config; + + readPreviousEpochsSpentAddresses(); + readLocalSpentAddresses(); + + return this; + } + + private void readPreviousEpochsSpentAddresses() { + if (config.isTestnet()) { + return; + } + + for (String previousEpochsSpentAddressesFile : config.getPreviousEpochSpentAddressesFiles().split(" ")) { + try { + readSpentAddressesFromStream(SpentAddressesProviderImpl.class.getResourceAsStream(previousEpochsSpentAddressesFile)); + } catch (SpentAddressesException e) { + log.error("failed to read spent addresses from " + previousEpochsSpentAddressesFile, e); + } + } + } + + private void readLocalSpentAddresses() { + String pathToLocalStateFile = config.getLocalSnapshotsBasePath() + ".snapshot.spentaddresses"; + try { + readSpentAddressesFromStream(new FileInputStream(config.getLocalSnapshotsBasePath() + ".snapshot.spentaddresses")); + } catch (Exception e) { + log.error("failed to read spent addresses from " + pathToLocalStateFile, e); + } + } + + private void readSpentAddressesFromStream(InputStream in) throws SpentAddressesException { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { + String line; + while ((line = reader.readLine()) != null) { + addAddress(HashFactory.ADDRESS.create(line)); + } + } catch (Exception e) { + throw new SpentAddressesException(e); + } + } + + @Override + public boolean containsAddress(Hash addressHash) throws SpentAddressesException { + try { + return ((SpentAddress) tangle.load(SpentAddress.class, addressHash)).exists; + } catch (Exception e) { + throw new SpentAddressesException(e); + } + } + + @Override + public void addAddress(Hash addressHash) throws SpentAddressesException { + try { + tangle.save(new SpentAddress(), addressHash); + } catch (Exception e) { + throw new SpentAddressesException(e); + } + } + + @Override + public void writeSpentAddressesToDisk(String basePath) { + // TODO: iterate over db and dump file + } } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java deleted file mode 100644 index e3e9a972dd..0000000000 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.iota.iri.service.spentaddresses.impl; - -import com.iota.iri.service.spentaddresses.SpentAddressesService; - -public class SpentAddressesServiceImpl implements SpentAddressesService { -} diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java index aa06a591f0..9398255286 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java @@ -1,13 +1,7 @@ package com.iota.iri.storage.rocksDB; import com.iota.iri.model.*; -import com.iota.iri.model.persistables.Address; -import com.iota.iri.model.persistables.Approvee; -import com.iota.iri.model.persistables.Bundle; -import com.iota.iri.model.persistables.Milestone; -import com.iota.iri.model.persistables.ObsoleteTag; -import com.iota.iri.model.persistables.Tag; -import com.iota.iri.model.persistables.Transaction; +import com.iota.iri.model.persistables.*; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.PersistenceProvider; @@ -35,6 +29,7 @@ public class RocksDBPersistenceProvider implements PersistenceProvider { private final List columnFamilyNames = Arrays.asList( new String(RocksDB.DEFAULT_COLUMN_FAMILY), + "spentAddress", "transaction", "transaction-metadata", "milestone", @@ -53,6 +48,7 @@ public class RocksDBPersistenceProvider implements PersistenceProvider { private final String logPath; private final int cacheSize; + private ColumnFamilyHandle spentAddressHandle; private ColumnFamilyHandle transactionHandle; private ColumnFamilyHandle transactionMetadataHandle; private ColumnFamilyHandle milestoneHandle; @@ -94,6 +90,7 @@ public boolean isAvailable() { private void initClassTreeMap() { Map, ColumnFamilyHandle> classMap = new LinkedHashMap<>(); + classMap.put(SpentAddress.class, spentAddressHandle); classMap.put(Transaction.class, transactionHandle); classMap.put(Milestone.class, milestoneHandle); classMap.put(StateDiff.class, stateDiffHandle); @@ -495,6 +492,7 @@ private void initDB(String path, String logPath) { private void fillModelColumnHandles() throws Exception { int i = 0; + spentAddressHandle = columnFamilyHandles.get(++i); transactionHandle = columnFamilyHandles.get(++i); transactionMetadataHandle = columnFamilyHandles.get(++i); milestoneHandle = columnFamilyHandles.get(++i); From 1553514f19cbfcd4147df2897ed0a686cc60bd91 Mon Sep 17 00:00:00 2001 From: Hans Moog Date: Fri, 21 Dec 2018 11:42:21 +0100 Subject: [PATCH 07/63] Feat: added SpentAddressesService --- src/main/java/com/iota/iri/Iota.java | 5 + src/main/java/com/iota/iri/service/API.java | 37 +------ .../spentaddresses/SpentAddressesService.java | 7 ++ .../impl/SpentAddressesServiceImpl.java | 97 +++++++++++++++++++ 4 files changed, 110 insertions(+), 36 deletions(-) create mode 100644 src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java create mode 100644 src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index a25abccdf6..7119bc74d4 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -21,6 +21,7 @@ import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; import com.iota.iri.service.snapshot.impl.SnapshotServiceImpl; import com.iota.iri.service.spentaddresses.impl.SpentAddressesProviderImpl; +import com.iota.iri.service.spentaddresses.impl.SpentAddressesServiceImpl; import com.iota.iri.service.tipselection.EntryPointSelector; import com.iota.iri.service.tipselection.RatingCalculator; import com.iota.iri.service.tipselection.TailFinder; @@ -84,6 +85,8 @@ public class Iota { public final SpentAddressesProviderImpl spentAddressesProvider; + public final SpentAddressesServiceImpl spentAddressesService; + public final SnapshotProviderImpl snapshotProvider; public final SnapshotServiceImpl snapshotService; @@ -131,6 +134,7 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh // new refactored instances spentAddressesProvider = new SpentAddressesProviderImpl(); + spentAddressesService = new SpentAddressesServiceImpl(); snapshotProvider = new SnapshotProviderImpl(); snapshotService = new SnapshotServiceImpl(); localSnapshotManager = configuration.getLocalSnapshotsEnabled() @@ -207,6 +211,7 @@ public void init() throws Exception { private void injectDependencies() throws SnapshotException, TransactionPruningException { spentAddressesProvider.init(tangle, configuration); + spentAddressesService.init(tangle, snapshotProvider, spentAddressesProvider); snapshotProvider.init(configuration); snapshotService.init(tangle, snapshotProvider, configuration); if (localSnapshotManager != null) { diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index f4ffccd618..932310f0c8 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -497,46 +497,11 @@ private AbstractResponse wereAddressesSpentFromStatement(List addresses) int index = 0; for (Hash address : addressesHash) { - states[index++] = wasAddressSpentFrom(address); + states[index++] = instance.spentAddressesService.wasAddressSpentFrom(address); } return WereAddressesSpentFrom.create(states); } - /** - * Checks if the address was ever spent from, in the current epoch, or in previous epochs. - * If an address has a pending transaction, it is also marked as spent. - * - * @param address The address to check if it was ever spent from. - * @return true if it was spent from, otherwise false - * @throws Exception When a model could not be loaded. - */ - private boolean wasAddressSpentFrom(Hash address) throws Exception { - if (instance.spentAddressesProvider.containsAddress(address)) { - return true; - } - - Set hashes = AddressViewModel.load(instance.tangle, address).getHashes(); - for (Hash hash : hashes) { - final TransactionViewModel tx = TransactionViewModel.fromHash(instance.tangle, hash); - // Check for spending transactions - if (tx.value() < 0) { - // Transaction is confirmed - if (tx.snapshotIndex() != 0) { - return true; - } - - // Transaction is pending - Hash tail = findTail(hash); - if (tail != null && BundleValidator.validate(instance.tangle, instance.snapshotProvider.getInitialSnapshot(), tail).size() != 0) { - return true; - } - } - } - - // No spending transaction found - return false; - } - /** * Walks back from the hash until a tail transaction has been found or transaction aprovee is not found. * A tail transaction is the first transaction in a bundle, thus with index = 0 diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java new file mode 100644 index 0000000000..e8a5c70124 --- /dev/null +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java @@ -0,0 +1,7 @@ +package com.iota.iri.service.spentaddresses; + +import com.iota.iri.model.Hash; + +public interface SpentAddressesService { + boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesException; +} diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java new file mode 100644 index 0000000000..00a251316d --- /dev/null +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -0,0 +1,97 @@ +package com.iota.iri.service.spentaddresses.impl; + +import com.iota.iri.BundleValidator; +import com.iota.iri.controllers.AddressViewModel; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.spentaddresses.SpentAddressesException; +import com.iota.iri.service.spentaddresses.SpentAddressesProvider; +import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.storage.Tangle; + +import java.util.Set; + +public class SpentAddressesServiceImpl implements SpentAddressesService { + private Tangle tangle; + + private SnapshotProvider snapshotProvider; + + private SpentAddressesProvider spentAddressesProvider; + + public SpentAddressesServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SpentAddressesProvider spentAddressesProvider) { + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + this.spentAddressesProvider = spentAddressesProvider; + + return this; + } + + @Override + public boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesException { + if (spentAddressesProvider.containsAddress(addressHash)) { + return true; + } + + try { + Set hashes = AddressViewModel.load(tangle, addressHash).getHashes(); + for (Hash hash : hashes) { + final TransactionViewModel tx = TransactionViewModel.fromHash(tangle, hash); + // Check for spending transactions + if (tx.value() < 0) { + // Transaction is confirmed + if (tx.snapshotIndex() != 0) { + return true; + } + + // Transaction is pending + Hash tail = findTail(hash); + if (tail != null && BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), tail).size() != 0) { + return true; + } + } + } + } catch (Exception e) { + throw new SpentAddressesException(e); + } + + return false; + } + + /** + * Walks back from the hash until a tail transaction has been found or transaction aprovee is not found. + * A tail transaction is the first transaction in a bundle, thus with index = 0 + * + * @param hash The transaction hash where we start the search from. If this is a tail, its hash is returned. + * @return The transaction hash of the tail + * @throws Exception When a model could not be loaded. + */ + private Hash findTail(Hash hash) throws Exception { + TransactionViewModel tx = TransactionViewModel.fromHash(tangle, hash); + final Hash bundleHash = tx.getBundleHash(); + long index = tx.getCurrentIndex(); + boolean foundApprovee = false; + + // As long as the index is bigger than 0 and we are still traversing the same bundle + // If the hash we asked about is already a tail, this loop never starts + while (index-- > 0 && tx.getBundleHash().equals(bundleHash)) { + Set approvees = tx.getApprovers(tangle).getHashes(); + for (Hash approvee : approvees) { + TransactionViewModel nextTx = TransactionViewModel.fromHash(tangle, approvee); + if (nextTx.getBundleHash().equals(bundleHash)) { + tx = nextTx; + foundApprovee = true; + break; + } + } + if (!foundApprovee) { + break; + } + } + + if (tx.getCurrentIndex() == 0) { + return tx.getHash(); + } + return null; + } +} From e1a4b88a21ba9c69e6a85c76781340d4d0bbb2cb Mon Sep 17 00:00:00 2001 From: Hans Moog Date: Fri, 21 Dec 2018 11:59:04 +0100 Subject: [PATCH 08/63] Feat: added method to update the spent addresses --- .../spentaddresses/SpentAddressesService.java | 2 ++ .../impl/SpentAddressesServiceImpl.java | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java index e8a5c70124..c49b0714e5 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java @@ -4,4 +4,6 @@ public interface SpentAddressesService { boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesException; + + void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws Exception; } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java index 00a251316d..aca01344ea 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -2,6 +2,7 @@ import com.iota.iri.BundleValidator; import com.iota.iri.controllers.AddressViewModel; +import com.iota.iri.controllers.MilestoneViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.service.snapshot.SnapshotProvider; @@ -9,7 +10,9 @@ import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.storage.Tangle; +import com.iota.iri.utils.dag.DAGHelper; +import java.util.HashSet; import java.util.Set; public class SpentAddressesServiceImpl implements SpentAddressesService { @@ -58,6 +61,31 @@ public boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesExcept return false; } + @Override + public void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws SpentAddressesException { + Set addressesToCheck = new HashSet<>(); + try { + for (int i = fromMilestoneIndex; i < toMilestoneIndex; i++) { + MilestoneViewModel currentMilestone = MilestoneViewModel.get(tangle, i); + if (currentMilestone != null) { + DAGHelper.get(tangle).traverseApprovees( + currentMilestone.getHash(), + transactionViewModel -> transactionViewModel.snapshotIndex() >= currentMilestone.index(), + transactionViewModel -> addressesToCheck.add(transactionViewModel.getAddressHash()) + ); + } + } + } catch (Exception e) { + throw new SpentAddressesException(e); + } + + for (Hash address : addressesToCheck) { + if (wasAddressSpentFrom(address)) { + spentAddressesProvider.addAddress(address); + } + } + } + /** * Walks back from the hash until a tail transaction has been found or transaction aprovee is not found. * A tail transaction is the first transaction in a bundle, thus with index = 0 From 055feb2a5cc98b26e89c97130a970ae7e9bc6653 Mon Sep 17 00:00:00 2001 From: Hans Moog Date: Fri, 21 Dec 2018 12:12:47 +0100 Subject: [PATCH 09/63] Feat: added the new classes to the snapshotting logic --- src/main/java/com/iota/iri/Iota.java | 2 +- .../snapshot/impl/SnapshotServiceImpl.java | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 7119bc74d4..058e7950c2 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -213,7 +213,7 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx spentAddressesProvider.init(tangle, configuration); spentAddressesService.init(tangle, snapshotProvider, spentAddressesProvider); snapshotProvider.init(configuration); - snapshotService.init(tangle, snapshotProvider, configuration); + snapshotService.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, configuration); if (localSnapshotManager != null) { localSnapshotManager.init(snapshotProvider, snapshotService, transactionPruner, configuration); } diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java index 51bbed011a..0e06d749e6 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java @@ -12,6 +12,8 @@ import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.snapshot.SnapshotException; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.spentaddresses.SpentAddressesProvider; +import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.service.transactionpruning.jobs.MilestonePrunerJob; @@ -81,6 +83,10 @@ public class SnapshotServiceImpl implements SnapshotService { */ private SnapshotConfig config; + private SpentAddressesService spentAddressesService; + + private SpentAddressesProvider spentAddressesProvider; + /** * This method initializes the instance and registers its dependencies.
*
@@ -98,9 +104,14 @@ public class SnapshotServiceImpl implements SnapshotService { * @param config important snapshot related configuration parameters * @return the initialized instance itself to allow chaining */ - public SnapshotServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SnapshotConfig config) { + public SnapshotServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, + SpentAddressesService spentAddressesService, SpentAddressesProvider spentAddressesProvider, + SnapshotConfig config) { + this.tangle = tangle; this.snapshotProvider = snapshotProvider; + this.spentAddressesService = spentAddressesService; + this.spentAddressesProvider = spentAddressesProvider; this.config = config; return this; @@ -488,6 +499,16 @@ private void cleanupOldData(SnapshotConfig config, TransactionPruner transaction private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot newSnapshot, SnapshotConfig config) throws SnapshotException { + try { + spentAddressesService.calculateSpentAddresses(snapshotProvider.getInitialSnapshot().getIndex(), + newSnapshot.getIndex()); + + spentAddressesProvider.writeSpentAddressesToDisk(config.getLocalSnapshotsBasePath() + + ".snapshot.spentaddresses"); + } catch (Exception e) { + throw new SnapshotException(e); + } + snapshotProvider.writeSnapshotToDisk(newSnapshot, config.getLocalSnapshotsBasePath()); snapshotProvider.getLatestSnapshot().lockWrite(); From e365c4ec5e84ec42b57c376922a37f52f66318eb Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Fri, 21 Dec 2018 16:15:05 +0200 Subject: [PATCH 10/63] Implement writeSpentAddressesToDisk with no append --- .../SpentAddressesProvider.java | 2 +- .../impl/SpentAddressesProviderImpl.java | 31 ++++++++++++++----- .../iota/iri/storage/PersistenceProvider.java | 2 ++ .../java/com/iota/iri/storage/Tangle.java | 24 ++++++++++++-- .../rocksDB/RocksDBPersistenceProvider.java | 29 ++++++++++++----- 5 files changed, 71 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java index d33d41c785..fc4cab72ae 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java @@ -7,5 +7,5 @@ public interface SpentAddressesProvider { void addAddress(Hash addressHash) throws SpentAddressesException; - void writeSpentAddressesToDisk(String basePath); + void writeSpentAddressesToDisk(String basePath) throws SpentAddressesException; } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index f815fbde75..03a9966789 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -7,13 +7,18 @@ import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import com.iota.iri.storage.Tangle; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.*; +import java.util.Collection; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SpentAddressesProviderImpl implements SpentAddressesProvider { private static final Logger log = LoggerFactory.getLogger(SpentAddressesProviderImpl.class); + private static final String SNAPSHOT_SPENTADDRESSES_FILE = ".snapshot.spentaddresses"; private Tangle tangle; @@ -38,7 +43,8 @@ private void readPreviousEpochsSpentAddresses() { for (String previousEpochsSpentAddressesFile : config.getPreviousEpochSpentAddressesFiles().split(" ")) { try { - readSpentAddressesFromStream(SpentAddressesProviderImpl.class.getResourceAsStream(previousEpochsSpentAddressesFile)); + readSpentAddressesFromStream( + SpentAddressesProviderImpl.class.getResourceAsStream(previousEpochsSpentAddressesFile)); } catch (SpentAddressesException e) { log.error("failed to read spent addresses from " + previousEpochsSpentAddressesFile, e); } @@ -46,9 +52,10 @@ private void readPreviousEpochsSpentAddresses() { } private void readLocalSpentAddresses() { - String pathToLocalStateFile = config.getLocalSnapshotsBasePath() + ".snapshot.spentaddresses"; + String pathToLocalStateFile = config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE; try { - readSpentAddressesFromStream(new FileInputStream(config.getLocalSnapshotsBasePath() + ".snapshot.spentaddresses")); + readSpentAddressesFromStream( + new FileInputStream(config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE)); } catch (Exception e) { log.error("failed to read spent addresses from " + pathToLocalStateFile, e); } @@ -84,7 +91,17 @@ public void addAddress(Hash addressHash) throws SpentAddressesException { } @Override - public void writeSpentAddressesToDisk(String basePath) { - // TODO: iterate over db and dump file + public void writeSpentAddressesToDisk(String basePath) throws SpentAddressesException { + try { + Collection addressHashes = getAllSpentAddresses(); + File snapshotFile = new File(config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE); + FileUtils.writeLines(snapshotFile, addressHashes, false); + } catch (Exception e) { + throw new SpentAddressesException("Failed to dump spent addresses to disk", e); + } + } + + private List getAllSpentAddresses() { + return tangle.loadAllKeysFromTable(SpentAddress.class, HashFactory.ADDRESS::create); } } diff --git a/src/main/java/com/iota/iri/storage/PersistenceProvider.java b/src/main/java/com/iota/iri/storage/PersistenceProvider.java index 24a6f58bc3..11e24a3024 100644 --- a/src/main/java/com/iota/iri/storage/PersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/PersistenceProvider.java @@ -51,4 +51,6 @@ public interface PersistenceProvider { void clear(Class column) throws Exception; void clearMetadata(Class column) throws Exception; + + List loadAllKeysFromTable(Class model); } diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index 2e5c515afe..73b994d9d3 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -1,13 +1,16 @@ package com.iota.iri.storage; import com.iota.iri.utils.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Created by paul on 3/3/17 for iri. @@ -123,6 +126,23 @@ public Set keysStartingWith(Class modelClass, byte[] value) { return output; } + public List loadAllKeysFromTable(Class modelClass, + Function transformer) { + List keys = null; + for(PersistenceProvider provider: this.persistenceProviders) { + if ((keys = provider.loadAllKeysFromTable(modelClass)) != null) { + break; + } + } + + if (keys != null) { + return keys.stream() + .map(transformer) + .collect(Collectors.toList()); + } + return null; + } + public Boolean exists(Class modelClass, Indexable hash) throws Exception { for(PersistenceProvider provider: this.persistenceProviders) { if (provider.exists(modelClass, hash)) { diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java index 9398255286..3b3cc70b46 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java @@ -1,18 +1,13 @@ package com.iota.iri.storage.rocksDB; -import com.iota.iri.model.*; +import com.iota.iri.model.HashFactory; +import com.iota.iri.model.StateDiff; import com.iota.iri.model.persistables.*; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.PersistenceProvider; import com.iota.iri.utils.IotaIOUtils; import com.iota.iri.utils.Pair; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.SystemUtils; -import org.rocksdb.*; -import org.rocksdb.util.SizeUnit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.nio.file.Paths; @@ -20,6 +15,13 @@ import java.util.*; import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.SystemUtils; +import org.rocksdb.*; +import org.rocksdb.util.SizeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class RocksDBPersistenceProvider implements PersistenceProvider { private static final Logger log = LoggerFactory.getLogger(RocksDBPersistenceProvider.class); @@ -360,6 +362,19 @@ public void clearMetadata(Class column) throws Exception { flushHandle(metadataReference.get(column)); } + @Override + public List loadAllKeysFromTable(Class column) { + List keyBytes = new ArrayList<>(); + + ColumnFamilyHandle columnFamilyHandle = classTreeMap.get(column); + try (RocksIterator iterator = db.newIterator(columnFamilyHandle)) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + keyBytes.add(iterator.key()); + } + } + return keyBytes; + } + private void flushHandle(ColumnFamilyHandle handle) throws RocksDBException { List itemsToDelete = new ArrayList<>(); try (RocksIterator iterator = db.newIterator(handle)) { From c7f7c49b51bac11dea0b94ece4d1e0ba75f1bc7c Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Fri, 21 Dec 2018 16:31:08 +0200 Subject: [PATCH 11/63] Create local snapshot addresses file if missing --- .../impl/SpentAddressesProviderImpl.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 03a9966789..ce534bc2ca 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -24,14 +24,26 @@ public class SpentAddressesProviderImpl implements SpentAddressesProvider { private IotaConfig config; + private File localSnapshotAddressesFile; + public SpentAddressesProviderImpl() {} - public SpentAddressesProviderImpl init(Tangle tangle, IotaConfig config) { + public SpentAddressesProviderImpl init(Tangle tangle, IotaConfig config) throws SpentAddressesException { this.tangle = tangle; this.config = config; + this.localSnapshotAddressesFile = new File(config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE); - readPreviousEpochsSpentAddresses(); readLocalSpentAddresses(); + if (localSnapshotAddressesFile.exists()) { + readPreviousEpochsSpentAddresses(); + } + else { + try { + localSnapshotAddressesFile.createNewFile(); + } catch (IOException e) { + throw new SpentAddressesException("Failed to create missing " + localSnapshotAddressesFile.getName(), e); + } + } return this; } @@ -52,12 +64,12 @@ private void readPreviousEpochsSpentAddresses() { } private void readLocalSpentAddresses() { - String pathToLocalStateFile = config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE; + try { readSpentAddressesFromStream( - new FileInputStream(config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE)); + new FileInputStream(localSnapshotAddressesFile)); } catch (Exception e) { - log.error("failed to read spent addresses from " + pathToLocalStateFile, e); + log.error("failed to read spent addresses from " + localSnapshotAddressesFile.getPath(), e); } } @@ -94,8 +106,7 @@ public void addAddress(Hash addressHash) throws SpentAddressesException { public void writeSpentAddressesToDisk(String basePath) throws SpentAddressesException { try { Collection addressHashes = getAllSpentAddresses(); - File snapshotFile = new File(config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE); - FileUtils.writeLines(snapshotFile, addressHashes, false); + FileUtils.writeLines(localSnapshotAddressesFile, addressHashes, false); } catch (Exception e) { throw new SpentAddressesException("Failed to dump spent addresses to disk", e); } From 4e063e7ffd872174307743913d946f8368f12328 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Fri, 21 Dec 2018 16:56:42 +0200 Subject: [PATCH 12/63] handle exceptions --- src/main/java/com/iota/iri/Iota.java | 38 +++++++++------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 058e7950c2..9b20ea3131 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -6,49 +6,35 @@ import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.network.Node; import com.iota.iri.network.TransactionRequester; -import com.iota.iri.network.impl.TransactionRequesterWorkerImpl; import com.iota.iri.network.UDPReceiver; +import com.iota.iri.network.impl.TransactionRequesterWorkerImpl; import com.iota.iri.network.replicator.Replicator; import com.iota.iri.service.TipsSolidifier; import com.iota.iri.service.ledger.impl.LedgerServiceImpl; -import com.iota.iri.service.milestone.impl.LatestMilestoneTrackerImpl; -import com.iota.iri.service.milestone.impl.LatestSolidMilestoneTrackerImpl; -import com.iota.iri.service.milestone.impl.MilestoneServiceImpl; -import com.iota.iri.service.milestone.impl.MilestoneSolidifierImpl; -import com.iota.iri.service.milestone.impl.SeenMilestonesRetrieverImpl; +import com.iota.iri.service.milestone.impl.*; import com.iota.iri.service.snapshot.SnapshotException; import com.iota.iri.service.snapshot.impl.LocalSnapshotManagerImpl; import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; import com.iota.iri.service.snapshot.impl.SnapshotServiceImpl; +import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.spentaddresses.impl.SpentAddressesProviderImpl; import com.iota.iri.service.spentaddresses.impl.SpentAddressesServiceImpl; -import com.iota.iri.service.tipselection.EntryPointSelector; -import com.iota.iri.service.tipselection.RatingCalculator; -import com.iota.iri.service.tipselection.TailFinder; -import com.iota.iri.service.tipselection.TipSelector; -import com.iota.iri.service.tipselection.Walker; -import com.iota.iri.service.tipselection.impl.CumulativeWeightCalculator; -import com.iota.iri.service.tipselection.impl.EntryPointSelectorImpl; -import com.iota.iri.service.tipselection.impl.TailFinderImpl; -import com.iota.iri.service.tipselection.impl.TipSelectorImpl; -import com.iota.iri.service.tipselection.impl.WalkerAlpha; +import com.iota.iri.service.tipselection.*; +import com.iota.iri.service.tipselection.impl.*; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; -import com.iota.iri.storage.Indexable; -import com.iota.iri.storage.Persistable; -import com.iota.iri.storage.PersistenceProvider; -import com.iota.iri.storage.Tangle; -import com.iota.iri.storage.ZmqPublishProvider; +import com.iota.iri.storage.*; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Pair; import com.iota.iri.zmq.MessageQ; -import org.apache.commons.lang3.NotImplementedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.security.SecureRandom; import java.util.List; +import org.apache.commons.lang3.NotImplementedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * * The main class of IRI. This will propagate transactions into and throughout the network. @@ -129,7 +115,7 @@ public class Iota { * @throws SnapshotException If the Snapshot fails to initialize. * This can happen if the snapshot signature is invalid or the file cannot be read. */ - public Iota(IotaConfig configuration) throws TransactionPruningException, SnapshotException { + public Iota(IotaConfig configuration) throws TransactionPruningException, SnapshotException, SpentAddressesException { this.configuration = configuration; // new refactored instances @@ -209,7 +195,7 @@ public void init() throws Exception { } } - private void injectDependencies() throws SnapshotException, TransactionPruningException { + private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException { spentAddressesProvider.init(tangle, configuration); spentAddressesService.init(tangle, snapshotProvider, spentAddressesProvider); snapshotProvider.init(configuration); From 3f2a23700b99f71679e163d6061c403db57150d5 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Fri, 21 Dec 2018 16:57:15 +0200 Subject: [PATCH 13/63] Implement reuqired method in zmq publisher --- .../java/com/iota/iri/storage/ZmqPublishProvider.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/storage/ZmqPublishProvider.java b/src/main/java/com/iota/iri/storage/ZmqPublishProvider.java index 9d226228c9..9425805933 100644 --- a/src/main/java/com/iota/iri/storage/ZmqPublishProvider.java +++ b/src/main/java/com/iota/iri/storage/ZmqPublishProvider.java @@ -6,13 +6,14 @@ import com.iota.iri.utils.Converter; import com.iota.iri.utils.Pair; import com.iota.iri.zmq.MessageQ; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.List; import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class ZmqPublishProvider implements PersistenceProvider { private static final Logger log = LoggerFactory.getLogger(ZmqPublishProvider.class); @@ -179,4 +180,9 @@ public void clear(Class column) throws Exception { public void clearMetadata(Class column) throws Exception { } + + @Override + public List loadAllKeysFromTable(Class model) { + return null; + } } \ No newline at end of file From cb4c91f2137d3b0955989536137c0e19d906c0dc Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Fri, 21 Dec 2018 18:15:44 +0200 Subject: [PATCH 14/63] check for existence of localSpentAddresses file properly --- .../spentaddresses/impl/SpentAddressesProviderImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index ce534bc2ca..37ce375547 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -33,9 +33,9 @@ public SpentAddressesProviderImpl init(Tangle tangle, IotaConfig config) throws this.config = config; this.localSnapshotAddressesFile = new File(config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE); - readLocalSpentAddresses(); + readPreviousEpochsSpentAddresses(); if (localSnapshotAddressesFile.exists()) { - readPreviousEpochsSpentAddresses(); + readLocalSpentAddresses(); } else { try { From a62b1d97c647fabdd376c343cb5905d90dbf22cd Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Fri, 21 Dec 2018 21:05:41 +0100 Subject: [PATCH 15/63] added batch spent address storage, added javadoc --- src/main/java/com/iota/iri/Iota.java | 5 +- .../SpentAddressesProvider.java | 35 +++++++++- .../spentaddresses/SpentAddressesService.java | 19 +++++ .../impl/SpentAddressesProviderImpl.java | 49 ++++++++++++- .../impl/SpentAddressesServiceImpl.java | 70 +++++++++++-------- 5 files changed, 143 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 9b20ea3131..1919204c23 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -23,7 +23,10 @@ import com.iota.iri.service.tipselection.impl.*; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; -import com.iota.iri.storage.*; +import com.iota.iri.storage.Indexable; +import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.Tangle; +import com.iota.iri.storage.ZmqPublishProvider; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Pair; import com.iota.iri.zmq.MessageQ; diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java index fc4cab72ae..213584f711 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java @@ -1,11 +1,44 @@ package com.iota.iri.service.spentaddresses; +import java.util.List; + import com.iota.iri.model.Hash; +/** + * Find, mark and store spent addresses + */ public interface SpentAddressesProvider { + + /** + * Checks if this address hash has been spent from + * + * @param addressHash The address to check for + * @return true if it is, else false + * @throws SpentAddressesException If the provider fails to check the address + */ boolean containsAddress(Hash addressHash) throws SpentAddressesException; + /** + * Mark an address as spent. + * + * @param addressHash the address which we want to mark. + * @throws SpentAddressesException If the provider fails to add the address + */ void addAddress(Hash addressHash) throws SpentAddressesException; - + + /** + * Mark all addresses as spent. + * + * @param addressHashes The addresses we want to mark + * @throws SpentAddressesException If the provider fails to add an address + */ + void addAddressesBatch(List addressHashes) throws SpentAddressesException; + + /** + * Writes all currently known spent addresses to disk. + * + * @param basePath the base path of the file we are going to write to. + * @throws SpentAddressesException If the provider fails to write the addresses to disk + */ void writeSpentAddressesToDisk(String basePath) throws SpentAddressesException; } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java index c49b0714e5..14b1a33d13 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java @@ -2,8 +2,27 @@ import com.iota.iri.model.Hash; +/** + * + * Check and calculate spent addresses + * + */ public interface SpentAddressesService { + + /** + * + * @param addressHash + * @return true if it was, else false + * @throws SpentAddressesException + */ boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesException; + /** + * Calculate all spent addresses in between a range + * + * @param fromMilestoneIndex the lower bound milestone index (inclusive) + * @param toMilestoneIndex the upper bound milestone index (exclusive) + * @throws Exception when anything went wrong whilst calculating. + */ void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws Exception; } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 37ce375547..c02cff7a63 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -1,34 +1,61 @@ package com.iota.iri.service.spentaddresses.impl; import com.iota.iri.conf.IotaConfig; +import com.iota.iri.conf.SnapshotConfig; import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; import com.iota.iri.model.persistables.SpentAddress; import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.spentaddresses.SpentAddressesProvider; +import com.iota.iri.storage.Indexable; +import com.iota.iri.storage.Persistable; import com.iota.iri.storage.Tangle; +import com.iota.iri.utils.Pair; import java.io.*; import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * + * Implementation of SpentAddressesProvider. + * Addresses are saved/found on the {@link Tangle}. + * The addresses will be written to a file called {@value #SNAPSHOT_SPENTADDRESSES_FILE}. + * The folder location is provided by {@link IotaConfig#getLocalSnapshotsBasePath()} + * + */ public class SpentAddressesProviderImpl implements SpentAddressesProvider { private static final Logger log = LoggerFactory.getLogger(SpentAddressesProviderImpl.class); private static final String SNAPSHOT_SPENTADDRESSES_FILE = ".snapshot.spentaddresses"; private Tangle tangle; - private IotaConfig config; + private SnapshotConfig config; private File localSnapshotAddressesFile; - public SpentAddressesProviderImpl() {} + /** + * Creates a new instance of SpentAddressesProvider + */ + public SpentAddressesProviderImpl() { + + } - public SpentAddressesProviderImpl init(Tangle tangle, IotaConfig config) throws SpentAddressesException { + /** + * Starts the SpentAddressesProvider by reading the previous spent addresses from file. + * If {@value #SNAPSHOT_SPENTADDRESSES_FILE} already exists, these addresses will be read as well. + * + * @param tangle Tangle object which acts as a database interface + * @param config The snapshot configuration used for file location + * @return the current instance + * @throws SpentAddressesException if we failed to create a file at the designated location + */ + public SpentAddressesProviderImpl init(Tangle tangle, SnapshotConfig config) throws SpentAddressesException { this.tangle = tangle; this.config = config; this.localSnapshotAddressesFile = new File(config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE); @@ -101,6 +128,22 @@ public void addAddress(Hash addressHash) throws SpentAddressesException { throw new SpentAddressesException(e); } } + + @Override + public void addAddressesBatch(List addressHash) throws SpentAddressesException { + try { + // Its bytes are always new byte[0], therefore identical in storage + SpentAddress spentAddressModel = new SpentAddress(); + + tangle.saveBatch(addressHash + .stream() + .map(address -> new Pair(address, spentAddressModel)) + .collect(Collectors.toList()) + ); + } catch (Exception e) { + throw new SpentAddressesException(e); + } + } @Override public void writeSpentAddressesToDisk(String basePath) throws SpentAddressesException { diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java index aca01344ea..f6fa66173b 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -9,24 +9,44 @@ import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.service.tipselection.TailFinder; +import com.iota.iri.service.tipselection.impl.TailFinderImpl; import com.iota.iri.storage.Tangle; import com.iota.iri.utils.dag.DAGHelper; import java.util.HashSet; +import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; +/** + * + * Implementation of SpentAddressesService that calculates and checks spent addresses using the {@link Tangle} + * + */ public class SpentAddressesServiceImpl implements SpentAddressesService { private Tangle tangle; private SnapshotProvider snapshotProvider; private SpentAddressesProvider spentAddressesProvider; + + private TailFinder tailFinder; + /** + * Creates a Spent address service using the Tangle + * + * @param tangle Tangle object which is used to load models of addresses + * @param snapshotProvider {@link SnapshotProvider} to find the genesis, used to verify tails + * @param spentAddressesProvider Provider for loading/saving addresses to a database. + * @return this instance + */ public SpentAddressesServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, SpentAddressesProvider spentAddressesProvider) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; this.spentAddressesProvider = spentAddressesProvider; - + this.tailFinder = new TailFinderImpl(tangle); + return this; } @@ -79,9 +99,23 @@ public void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex throw new SpentAddressesException(e); } - for (Hash address : addressesToCheck) { - if (wasAddressSpentFrom(address)) { - spentAddressesProvider.addAddress(address); + //Can only throw runtime exceptions in streams + try { + spentAddressesProvider.addAddressesBatch(addressesToCheck.stream() + .filter(address -> { + try { + return wasAddressSpentFrom(address); + } catch (SpentAddressesException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList())); + + } catch (RuntimeException e) { + if (e.getCause() instanceof SpentAddressesException) { + throw (SpentAddressesException) e.getCause(); + } else { + throw e; } } } @@ -95,31 +129,7 @@ public void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex * @throws Exception When a model could not be loaded. */ private Hash findTail(Hash hash) throws Exception { - TransactionViewModel tx = TransactionViewModel.fromHash(tangle, hash); - final Hash bundleHash = tx.getBundleHash(); - long index = tx.getCurrentIndex(); - boolean foundApprovee = false; - - // As long as the index is bigger than 0 and we are still traversing the same bundle - // If the hash we asked about is already a tail, this loop never starts - while (index-- > 0 && tx.getBundleHash().equals(bundleHash)) { - Set approvees = tx.getApprovers(tangle).getHashes(); - for (Hash approvee : approvees) { - TransactionViewModel nextTx = TransactionViewModel.fromHash(tangle, approvee); - if (nextTx.getBundleHash().equals(bundleHash)) { - tx = nextTx; - foundApprovee = true; - break; - } - } - if (!foundApprovee) { - break; - } - } - - if (tx.getCurrentIndex() == 0) { - return tx.getHash(); - } - return null; + Optional optionalTail = tailFinder.findTail(hash); + return optionalTail.isPresent() ? optionalTail.get() : null; } } From bad59e91077c6d355caf8c1ee65cd3bade9ae206 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 23 Dec 2018 10:52:16 +0200 Subject: [PATCH 16/63] bump version to 1.6.0-RC12 --- DOCKER.md | 6 +++--- pom.xml | 2 +- src/main/java/com/iota/iri/IRI.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index 1572259333..8375fd0999 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -2,14 +2,14 @@ Run the official iotaledger/iri container, passing the mandatory -p option: -```docker run iotaledger/iri:v1.6.0-RC11 -p 14265``` +```docker run iotaledger/iri:v1.6.0-RC12 -p 14265``` This will get your a running IRI with its API listening on port 14265, no neighbours and an empty database. The IRI Docker container by default expects data at /iri/data. Use the `-v` option of the `docker run` command to mount volumes so to have persistent data. You can also pass more command line options to the docker run command and those will be passed to IRI. If you want to use a iri.ini file with the docker container, supposing it's stored under /path/to/conf/iri.ini on your docker host, then pass `-v /path/to/conf:/iri/conf` and add -c /iri/conf/iri.ini as docker run arguments. So for example the `docker run` command above would become: -```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC11 -p 14265 -c /iri/conf/iri.ini``` +```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC12 -p 14265 -c /iri/conf/iri.ini``` Please refer to the IRI documentation for further command line options and iri.ini options. @@ -62,7 +62,7 @@ ExecStart=/usr/bin/docker run \ -p 14265:14265 \ -p 15600:15600 \ -p 14600:14600/udp \ -iotaledger/iri:v1.6.0-RC11 \ +iotaledger/iri:v1.6.0-RC12 \ -p 14265 \ --zmq-enabled \ --testnet diff --git a/pom.xml b/pom.xml index 72fb4b9057..c99d4bab86 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.iota iri - 1.6.0-RC11 + 1.6.0-RC12 IRI IOTA Reference Implementation diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index 142c70196f..1aebd7a09e 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -42,7 +42,7 @@ public class IRI { public static final String MAINNET_NAME = "IRI"; public static final String TESTNET_NAME = "IRI Testnet"; - public static final String VERSION = "1.6.0-RC11"; + public static final String VERSION = "1.6.0-RC12"; /** * The entry point of IRI. From 47c4f5e2b57b40393b7e09c079c542ac21d4ef8d Mon Sep 17 00:00:00 2001 From: Hans Moog Date: Thu, 3 Jan 2019 11:24:44 +0100 Subject: [PATCH 17/63] Feat: write snapshot files to temp files first --- .../snapshot/impl/SnapshotProviderImpl.java | 30 +++++++++++++++++-- testnet.snapshot.spentaddresses | 0 2 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 testnet.snapshot.spentaddresses diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java index 69c163a351..65becf2b1e 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java @@ -9,6 +9,7 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -118,15 +119,38 @@ public Snapshot getLatestSnapshot() { } /** - * {@inheritDoc} + * {@inheritDoc}
+ *
+ * It first writes two temporary files and then renames them to avoid corruption of the written files in case of IRI + * crashes.
*/ @Override public void writeSnapshotToDisk(Snapshot snapshot, String basePath) throws SnapshotException { snapshot.lockRead(); + File tmpStateFile = null; + File tmpMetaFile = null; try { - writeSnapshotStateToDisk(snapshot, basePath + ".snapshot.state"); - writeSnapshotMetaDataToDisk(snapshot, basePath + ".snapshot.meta"); + tmpStateFile = File.createTempFile("iri.", ".snapshot.state"); + tmpMetaFile = File.createTempFile("iri.", ".snapshot.meta"); + + writeSnapshotStateToDisk(snapshot, tmpStateFile.getAbsolutePath()); + writeSnapshotMetaDataToDisk(snapshot, tmpMetaFile.getAbsolutePath()); + + Files.move(Paths.get(tmpStateFile.getAbsolutePath()), Paths.get(basePath + ".snapshot.state"), + StandardCopyOption.ATOMIC_MOVE); + Files.move(Paths.get(tmpMetaFile.getAbsolutePath()), Paths.get(basePath + ".snapshot.meta"), + StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + // issue a delete for both temp files - just to be safe and not pollute the temp folder + if (tmpStateFile != null) { + tmpStateFile.delete(); + } + if (tmpMetaFile != null) { + tmpMetaFile.delete(); + } + + throw new SnapshotException("failed to write snapshot files", e); } finally { snapshot.unlockRead(); } diff --git a/testnet.snapshot.spentaddresses b/testnet.snapshot.spentaddresses new file mode 100644 index 0000000000..e69de29bb2 From b6f89531776b027ff0a4ad4093eccbe67d632463 Mon Sep 17 00:00:00 2001 From: Alon Elmaliah Date: Thu, 3 Jan 2019 12:49:49 +0200 Subject: [PATCH 18/63] fix first() calls in view models --- .../iri/controllers/AddressViewModel.java | 3 ++- .../iri/controllers/ApproveeViewModel.java | 5 ++--- .../iota/iri/controllers/BundleViewModel.java | 5 ++--- .../iota/iri/controllers/TagViewModel.java | 5 ++--- .../iri/controllers/TransactionViewModel.java | 9 ++------- .../java/com/iota/iri/model/AbstractHash.java | 11 ++++++---- .../java/com/iota/iri/model/AddressHash.java | 2 ++ .../java/com/iota/iri/model/BundleHash.java | 2 ++ src/main/java/com/iota/iri/model/TagHash.java | 2 ++ .../com/iota/iri/model/TransactionHash.java | 4 +++- .../iri/controllers/BundleViewModelTest.java | 20 +++++++++++++++++++ .../controllers/TransactionViewModelTest.java | 10 ++++++++++ 12 files changed, 56 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/iota/iri/controllers/AddressViewModel.java b/src/main/java/com/iota/iri/controllers/AddressViewModel.java index b25aaa5fa5..77036578c4 100644 --- a/src/main/java/com/iota/iri/controllers/AddressViewModel.java +++ b/src/main/java/com/iota/iri/controllers/AddressViewModel.java @@ -1,5 +1,6 @@ package com.iota.iri.controllers; +import com.iota.iri.model.AddressHash; import com.iota.iri.model.Hash; import com.iota.iri.model.persistables.Address; import com.iota.iri.storage.Indexable; @@ -59,7 +60,7 @@ public static AddressViewModel load(Tangle tangle, Indexable hash) throws Except * @throws Exception Thrown if the database fails to return a first object */ public static AddressViewModel first(Tangle tangle) throws Exception { - Pair bundlePair = tangle.getFirst(Address.class, Hash.class); + Pair bundlePair = tangle.getFirst(Address.class, AddressHash.class); if(bundlePair != null && bundlePair.hi != null) { return new AddressViewModel((Address) bundlePair.hi, (Hash) bundlePair.low); } diff --git a/src/main/java/com/iota/iri/controllers/ApproveeViewModel.java b/src/main/java/com/iota/iri/controllers/ApproveeViewModel.java index 28ce1f627e..08bd4d1293 100644 --- a/src/main/java/com/iota/iri/controllers/ApproveeViewModel.java +++ b/src/main/java/com/iota/iri/controllers/ApproveeViewModel.java @@ -1,14 +1,13 @@ package com.iota.iri.controllers; import com.iota.iri.model.Hash; +import com.iota.iri.model.TransactionHash; import com.iota.iri.model.persistables.Approvee; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.Tangle; import com.iota.iri.utils.Pair; -import java.util.HashMap; -import java.util.Map; import java.util.Set; /** @@ -61,7 +60,7 @@ public static ApproveeViewModel load(Tangle tangle, Indexable hash) throws Excep * @throws Exception Thrown if the database fails to return a first object */ public static ApproveeViewModel first(Tangle tangle) throws Exception { - Pair bundlePair = tangle.getFirst(Approvee.class, Hash.class); + Pair bundlePair = tangle.getFirst(Approvee.class, TransactionHash.class); if(bundlePair != null && bundlePair.hi != null) { return new ApproveeViewModel((Approvee) bundlePair.hi, (Hash) bundlePair.low); } diff --git a/src/main/java/com/iota/iri/controllers/BundleViewModel.java b/src/main/java/com/iota/iri/controllers/BundleViewModel.java index 2b32bad82c..fa2f0d9f4d 100644 --- a/src/main/java/com/iota/iri/controllers/BundleViewModel.java +++ b/src/main/java/com/iota/iri/controllers/BundleViewModel.java @@ -1,5 +1,6 @@ package com.iota.iri.controllers; +import com.iota.iri.model.BundleHash; import com.iota.iri.model.Hash; import com.iota.iri.model.persistables.Bundle; import com.iota.iri.storage.Indexable; @@ -7,8 +8,6 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.utils.Pair; -import java.util.HashMap; -import java.util.Map; import java.util.Set; /** @@ -61,7 +60,7 @@ public static BundleViewModel load(Tangle tangle, Indexable hash) throws Excepti * @throws Exception Thrown if the database fails to return a first object */ public static BundleViewModel first(Tangle tangle) throws Exception { - Pair bundlePair = tangle.getFirst(Bundle.class, Hash.class); + Pair bundlePair = tangle.getFirst(Bundle.class, BundleHash.class); if(bundlePair != null && bundlePair.hi != null) { return new BundleViewModel((Bundle) bundlePair.hi, (Hash) bundlePair.low); } diff --git a/src/main/java/com/iota/iri/controllers/TagViewModel.java b/src/main/java/com/iota/iri/controllers/TagViewModel.java index 17a54c2f4d..fa535ee5eb 100644 --- a/src/main/java/com/iota/iri/controllers/TagViewModel.java +++ b/src/main/java/com/iota/iri/controllers/TagViewModel.java @@ -1,6 +1,7 @@ package com.iota.iri.controllers; import com.iota.iri.model.Hash; +import com.iota.iri.model.TagHash; import com.iota.iri.model.persistables.ObsoleteTag; import com.iota.iri.model.persistables.Tag; import com.iota.iri.storage.Indexable; @@ -8,8 +9,6 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.utils.Pair; -import java.util.HashMap; -import java.util.Map; import java.util.Set; /** @@ -94,7 +93,7 @@ public static TagViewModel loadObsolete(Tangle tangle, Indexable hash) throws Ex * @throws Exception Thrown if the database fails to return a first object */ public static TagViewModel first(Tangle tangle) throws Exception { - Pair tagPair = tangle.getFirst(Tag.class, Hash.class); + Pair tagPair = tangle.getFirst(Tag.class, TagHash.class); if(tagPair != null && tagPair.hi != null) { return new TagViewModel((Tag) tagPair.hi, (Hash) tagPair.low); } diff --git a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java index 2622025ff8..ed53da8dba 100644 --- a/src/main/java/com/iota/iri/controllers/TransactionViewModel.java +++ b/src/main/java/com/iota/iri/controllers/TransactionViewModel.java @@ -1,13 +1,8 @@ package com.iota.iri.controllers; import com.iota.iri.model.*; +import com.iota.iri.model.persistables.*; import com.iota.iri.service.snapshot.Snapshot; -import com.iota.iri.model.persistables.Address; -import com.iota.iri.model.persistables.Approvee; -import com.iota.iri.model.persistables.Bundle; -import com.iota.iri.model.persistables.ObsoleteTag; -import com.iota.iri.model.persistables.Tag; -import com.iota.iri.model.persistables.Transaction; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.Tangle; @@ -246,7 +241,7 @@ public static Set getMissingTransactions(Tangle tangle) throws Except * @throws Exception Thrown if the database fails to return a first object. */ public static TransactionViewModel first(Tangle tangle) throws Exception { - Pair transactionPair = tangle.getFirst(Transaction.class, Hash.class); + Pair transactionPair = tangle.getFirst(Transaction.class, TransactionHash.class); if (transactionPair != null && transactionPair.hi != null) { return new TransactionViewModel((Transaction) transactionPair.hi, (Hash) transactionPair.low); } diff --git a/src/main/java/com/iota/iri/model/AbstractHash.java b/src/main/java/com/iota/iri/model/AbstractHash.java index e3cd34c231..f85d7401e3 100644 --- a/src/main/java/com/iota/iri/model/AbstractHash.java +++ b/src/main/java/com/iota/iri/model/AbstractHash.java @@ -1,9 +1,5 @@ package com.iota.iri.model; -import java.io.Serializable; -import java.util.Arrays; -import java.util.Objects; - import com.iota.iri.crypto.Curl; import com.iota.iri.model.persistables.Transaction; import com.iota.iri.model.safe.ByteSafe; @@ -11,12 +7,19 @@ import com.iota.iri.storage.Indexable; import com.iota.iri.utils.Converter; +import java.io.Serializable; +import java.util.Arrays; +import java.util.Objects; + public abstract class AbstractHash implements Hash, Serializable { private final Object lock = new Object(); private ByteSafe byteSafe; private TritSafe tritSafe; + public AbstractHash() { + } + public AbstractHash(byte[] source, int sourceOffset, int sourceSize) { if(sourceSize < SIZE_IN_TRITS) { byte[] dest = new byte[SIZE_IN_BYTES]; diff --git a/src/main/java/com/iota/iri/model/AddressHash.java b/src/main/java/com/iota/iri/model/AddressHash.java index d036fa5d4c..e0fd559af8 100644 --- a/src/main/java/com/iota/iri/model/AddressHash.java +++ b/src/main/java/com/iota/iri/model/AddressHash.java @@ -2,6 +2,8 @@ public class AddressHash extends AbstractHash { + public AddressHash() { } + protected AddressHash(byte[] bytes, int offset, int sizeInBytes) { super(bytes, offset, sizeInBytes); } diff --git a/src/main/java/com/iota/iri/model/BundleHash.java b/src/main/java/com/iota/iri/model/BundleHash.java index 31af8f111e..dbfc4a9a25 100644 --- a/src/main/java/com/iota/iri/model/BundleHash.java +++ b/src/main/java/com/iota/iri/model/BundleHash.java @@ -2,6 +2,8 @@ public class BundleHash extends AbstractHash { + public BundleHash() { } + protected BundleHash(byte[] bytes, int offset, int sizeInBytes) { super(bytes, offset, sizeInBytes); } diff --git a/src/main/java/com/iota/iri/model/TagHash.java b/src/main/java/com/iota/iri/model/TagHash.java index 7e66005802..486736d704 100644 --- a/src/main/java/com/iota/iri/model/TagHash.java +++ b/src/main/java/com/iota/iri/model/TagHash.java @@ -2,6 +2,8 @@ public class TagHash extends AbstractHash { + public TagHash() { } + protected TagHash(byte[] tagBytes, int offset, int tagSizeInBytes) { super(tagBytes, offset, tagSizeInBytes); } diff --git a/src/main/java/com/iota/iri/model/TransactionHash.java b/src/main/java/com/iota/iri/model/TransactionHash.java index 5aa7523fea..6e0cb7ad95 100644 --- a/src/main/java/com/iota/iri/model/TransactionHash.java +++ b/src/main/java/com/iota/iri/model/TransactionHash.java @@ -6,7 +6,9 @@ public class TransactionHash extends AbstractHash { - protected TransactionHash(byte[] source, int offset, int sourceSize) { + public TransactionHash() { } + + protected TransactionHash(byte[] source, int offset, int sourceSize) { super(source, offset, sourceSize); } diff --git a/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java b/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java index b8dca71094..03d404061c 100644 --- a/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java @@ -1,8 +1,14 @@ package com.iota.iri.controllers; +import com.iota.iri.conf.MainnetConfig; +import com.iota.iri.crypto.SpongeFactory; +import com.iota.iri.model.TransactionHash; +import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -14,6 +20,7 @@ public class BundleViewModelTest { private static final TemporaryFolder dbFolder = new TemporaryFolder(); private static final TemporaryFolder logFolder = new TemporaryFolder(); private static Tangle tangle = new Tangle(); + private static SnapshotProvider snapshotProvider; @Before public void setUp() throws Exception { @@ -24,6 +31,7 @@ public void setUp() throws Exception { logFolder.getRoot().getAbsolutePath(),1000); tangle.addPersistenceProvider(rocksDBPersistenceProvider); tangle.init(); + snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); } @@ -32,6 +40,8 @@ public void tearDown() throws Exception { tangle.shutdown(); dbFolder.delete(); logFolder.delete(); + snapshotProvider.shutdown(); + } @Test @@ -59,4 +69,14 @@ public void getTail() throws Exception { } + @Test + public void firstShouldFindTx() throws Exception { + byte[] trits = TransactionViewModelTest.getRandomTransactionTrits(); + TransactionViewModel transactionViewModel = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); + transactionViewModel.store(tangle, snapshotProvider.getInitialSnapshot()); + + BundleViewModel result = BundleViewModel.first(tangle); + Assert.assertTrue(result.getHashes().contains(transactionViewModel.getHash())); + } + } \ No newline at end of file diff --git a/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java b/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java index b3ae7f91c6..5b8b893f92 100644 --- a/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java @@ -402,6 +402,16 @@ public void testManyTXInDB() throws Exception { log.info("Done. #TX: {}", TransactionViewModel.getNumberOfStoredTransactions(tangle)); } + @Test + public void firstShouldFindTx() throws Exception { + byte[] trits = getRandomTransactionTrits(); + TransactionViewModel transactionViewModel = new TransactionViewModel(trits, TransactionHash.calculate(SpongeFactory.Mode.CURLP81, trits)); + transactionViewModel.store(tangle, snapshotProvider.getInitialSnapshot()); + + TransactionViewModel result = TransactionViewModel.first(tangle); + Assert.assertEquals(transactionViewModel.getHash(), result.getHash()); + } + private Transaction getRandomTransaction(Random seed) { Transaction transaction = new Transaction(); From 3ede36acc763f4fc443327d58f2dc3da0c479124 Mon Sep 17 00:00:00 2001 From: Alon Elmaliah Date: Thu, 3 Jan 2019 13:02:57 +0200 Subject: [PATCH 19/63] decouple TxVM tests --- .../iri/controllers/TransactionViewModelTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java b/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java index 5b8b893f92..479601eabe 100644 --- a/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java @@ -11,9 +11,9 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Converter; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; @@ -36,8 +36,8 @@ public class TransactionViewModelTest { private static final Random seed = new Random(); - @BeforeClass - public static void setUp() throws Exception { + @Before + public void setUp() throws Exception { dbFolder.create(); logFolder.create(); RocksDBPersistenceProvider rocksDBPersistenceProvider; @@ -48,8 +48,8 @@ public static void setUp() throws Exception { snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); } - @AfterClass - public static void tearDown() throws Exception { + @After + public void tearDown() throws Exception { tangle.shutdown(); snapshotProvider.shutdown(); dbFolder.delete(); From 0f610c0dc40aa8689fb52dd1dadd587c41c1c400 Mon Sep 17 00:00:00 2001 From: Hans Moog Date: Thu, 3 Jan 2019 14:08:02 +0100 Subject: [PATCH 20/63] Refactor: rename now creates backup files (to force renamce over copy) --- .../snapshot/impl/SnapshotProviderImpl.java | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java index 65becf2b1e..4ed64b1bc4 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java @@ -121,35 +121,37 @@ public Snapshot getLatestSnapshot() { /** * {@inheritDoc}
*
- * It first writes two temporary files and then renames them to avoid corruption of the written files in case of IRI - * crashes.
+ * It first writes two temporary files, then renames the current files by appending them with a ".bkp" extension and + * finally renames the temporary files. This mechanism reduces the chances of the files getting corrupted if IRI + * crashes during the snapshot creation and always leaves the node operator with a set of backup files that can be + * renamed to resume node operation prior to the failed snapshot.
+ *
+ * Note: We create the temporary files in the same folder as the "real" files to allow the operating system to + * perform a "rename" instead of a "copy" operation.
*/ @Override public void writeSnapshotToDisk(Snapshot snapshot, String basePath) throws SnapshotException { snapshot.lockRead(); - File tmpStateFile = null; - File tmpMetaFile = null; try { - tmpStateFile = File.createTempFile("iri.", ".snapshot.state"); - tmpMetaFile = File.createTempFile("iri.", ".snapshot.meta"); - - writeSnapshotStateToDisk(snapshot, tmpStateFile.getAbsolutePath()); - writeSnapshotMetaDataToDisk(snapshot, tmpMetaFile.getAbsolutePath()); - - Files.move(Paths.get(tmpStateFile.getAbsolutePath()), Paths.get(basePath + ".snapshot.state"), - StandardCopyOption.ATOMIC_MOVE); - Files.move(Paths.get(tmpMetaFile.getAbsolutePath()), Paths.get(basePath + ".snapshot.meta"), - StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { - // issue a delete for both temp files - just to be safe and not pollute the temp folder - if (tmpStateFile != null) { - tmpStateFile.delete(); + // write new temp files + writeSnapshotStateToDisk(snapshot, basePath + ".snapshot.state.tmp"); + writeSnapshotMetaDataToDisk(snapshot, basePath + ".snapshot.meta.tmp"); + + // rename current files by appending ".bkp" + if (new File(basePath + ".snapshot.state").exists()) { + Files.move(Paths.get(basePath + ".snapshot.state"), Paths.get(basePath + ".snapshot.state.bkp"), + StandardCopyOption.REPLACE_EXISTING); } - if (tmpMetaFile != null) { - tmpMetaFile.delete(); + if (new File(basePath + ".snapshot.meta").exists()) { + Files.move(Paths.get(basePath + ".snapshot.meta"), Paths.get(basePath + ".snapshot.meta.bkp"), + StandardCopyOption.REPLACE_EXISTING); } + // rename temp files to their final name + Files.move(Paths.get(basePath + ".snapshot.state.tmp"), Paths.get(basePath + ".snapshot.state")); + Files.move(Paths.get(basePath + ".snapshot.meta.tmp"), Paths.get(basePath + ".snapshot.meta")); + } catch (IOException e) { throw new SnapshotException("failed to write snapshot files", e); } finally { snapshot.unlockRead(); From 8a9813eee764a633267997822671a790a9acde59 Mon Sep 17 00:00:00 2001 From: Hans Moog Date: Sun, 6 Jan 2019 19:26:14 +0100 Subject: [PATCH 21/63] Feat: Write snapshot files to temp files first (#1256) * Feat: write snapshot files to temp files first * Refactor: rename now creates backup files (to force renamce over copy) --- .../snapshot/impl/SnapshotProviderImpl.java | 32 +++++++++++++++++-- testnet.snapshot.spentaddresses | 0 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 testnet.snapshot.spentaddresses diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java index 69c163a351..4ed64b1bc4 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java @@ -9,6 +9,7 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -118,15 +119,40 @@ public Snapshot getLatestSnapshot() { } /** - * {@inheritDoc} + * {@inheritDoc}
+ *
+ * It first writes two temporary files, then renames the current files by appending them with a ".bkp" extension and + * finally renames the temporary files. This mechanism reduces the chances of the files getting corrupted if IRI + * crashes during the snapshot creation and always leaves the node operator with a set of backup files that can be + * renamed to resume node operation prior to the failed snapshot.
+ *
+ * Note: We create the temporary files in the same folder as the "real" files to allow the operating system to + * perform a "rename" instead of a "copy" operation.
*/ @Override public void writeSnapshotToDisk(Snapshot snapshot, String basePath) throws SnapshotException { snapshot.lockRead(); try { - writeSnapshotStateToDisk(snapshot, basePath + ".snapshot.state"); - writeSnapshotMetaDataToDisk(snapshot, basePath + ".snapshot.meta"); + // write new temp files + writeSnapshotStateToDisk(snapshot, basePath + ".snapshot.state.tmp"); + writeSnapshotMetaDataToDisk(snapshot, basePath + ".snapshot.meta.tmp"); + + // rename current files by appending ".bkp" + if (new File(basePath + ".snapshot.state").exists()) { + Files.move(Paths.get(basePath + ".snapshot.state"), Paths.get(basePath + ".snapshot.state.bkp"), + StandardCopyOption.REPLACE_EXISTING); + } + if (new File(basePath + ".snapshot.meta").exists()) { + Files.move(Paths.get(basePath + ".snapshot.meta"), Paths.get(basePath + ".snapshot.meta.bkp"), + StandardCopyOption.REPLACE_EXISTING); + } + + // rename temp files to their final name + Files.move(Paths.get(basePath + ".snapshot.state.tmp"), Paths.get(basePath + ".snapshot.state")); + Files.move(Paths.get(basePath + ".snapshot.meta.tmp"), Paths.get(basePath + ".snapshot.meta")); + } catch (IOException e) { + throw new SnapshotException("failed to write snapshot files", e); } finally { snapshot.unlockRead(); } diff --git a/testnet.snapshot.spentaddresses b/testnet.snapshot.spentaddresses new file mode 100644 index 0000000000..e69de29bb2 From 98954a58f54b9f4c616bfda1da13ba5fd8704c8b Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 6 Jan 2019 20:30:48 +0200 Subject: [PATCH 22/63] delete file that was erronously committed --- testnet.snapshot.spentaddresses | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 testnet.snapshot.spentaddresses diff --git a/testnet.snapshot.spentaddresses b/testnet.snapshot.spentaddresses deleted file mode 100644 index e69de29bb2..0000000000 From 0b8e0ec461c7df3430e8c9f19ed6b8fd3b929f0c Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 6 Jan 2019 11:48:58 +0200 Subject: [PATCH 23/63] Revert "disable local-snapshots" This reverts commit 793258d --- src/main/java/com/iota/iri/Iota.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index a268fc3667..cd7996b5f6 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -23,11 +23,7 @@ import com.iota.iri.service.tipselection.impl.*; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; -import com.iota.iri.storage.Indexable; -import com.iota.iri.storage.Persistable; -import com.iota.iri.storage.PersistenceProvider; -import com.iota.iri.storage.Tangle; -import com.iota.iri.storage.ZmqPublishProvider; +import com.iota.iri.storage.*; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Pair; import com.iota.iri.zmq.MessageQ; @@ -127,13 +123,17 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh spentAddressesService = new SpentAddressesServiceImpl(); snapshotProvider = new SnapshotProviderImpl(); snapshotService = new SnapshotServiceImpl(); - localSnapshotManager = null; + localSnapshotManager = configuration.getLocalSnapshotsEnabled() + ? new LocalSnapshotManagerImpl() + : null; milestoneService = new MilestoneServiceImpl(); latestMilestoneTracker = new LatestMilestoneTrackerImpl(); latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl(); seenMilestonesRetriever = new SeenMilestonesRetrieverImpl(); milestoneSolidifier = new MilestoneSolidifierImpl(); - transactionPruner = null; + transactionPruner = configuration.getLocalSnapshotsEnabled() && configuration.getLocalSnapshotsPruningEnabled() + ? new AsyncTransactionPruner() + : null; transactionRequesterWorker = new TransactionRequesterWorkerImpl(); // legacy code @@ -195,9 +195,7 @@ public void init() throws Exception { } } - private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException { - spentAddressesProvider.init(tangle, configuration); - spentAddressesService.init(tangle, snapshotProvider, spentAddressesProvider); + private void injectDependencies() throws SnapshotException, TransactionPruningException { snapshotProvider.init(configuration); snapshotService.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, configuration); if (localSnapshotManager != null) { From d2119ca585dab623481990408097a2bda52706ed Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 6 Jan 2019 20:20:11 +0200 Subject: [PATCH 24/63] pom: add throwing-function dependency --- pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pom.xml b/pom.xml index 3b0fbbf112..f42994315d 100644 --- a/pom.xml +++ b/pom.xml @@ -197,6 +197,12 @@ test + + pl.touk + throwing-function + 1.3 + + @@ -372,6 +378,9 @@ com.beust:jcommander:1.72:jar:null:compile:6375e521c1e11d6563d4f25a07ce124ccf8cd171 + + pl.touk:throwing-function:1.3:jar:null:compile:32947866b8754295efde73ee7d39ea29a247a2b5 + From 423f1367a965a8db84161c33b6efb6d22eb7c7bf Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 6 Jan 2019 20:21:18 +0200 Subject: [PATCH 25/63] SpentAddresses: persist addresses from a given list of transactions --- .../snapshot/impl/SnapshotServiceImpl.java | 17 ++--- .../SpentAddressesProvider.java | 6 +- .../spentaddresses/SpentAddressesService.java | 14 +++- .../impl/SpentAddressesProviderImpl.java | 2 +- .../impl/SpentAddressesServiceImpl.java | 72 ++++++++++++------- .../iri/service/tipselection/TailFinder.java | 16 +++-- .../tipselection/impl/TailFinderImpl.java | 4 ++ .../tipselection/impl/WalkerAlphaTest.java | 21 ++++-- 8 files changed, 99 insertions(+), 53 deletions(-) diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java index 0e06d749e6..40a865fa70 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java @@ -7,11 +7,7 @@ import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.service.milestone.LatestMilestoneTracker; -import com.iota.iri.service.snapshot.SnapshotMetaData; -import com.iota.iri.service.snapshot.SnapshotService; -import com.iota.iri.service.snapshot.Snapshot; -import com.iota.iri.service.snapshot.SnapshotException; -import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.snapshot.*; import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPruner; @@ -19,12 +15,10 @@ import com.iota.iri.service.transactionpruning.jobs.MilestonePrunerJob; import com.iota.iri.service.transactionpruning.jobs.UnconfirmedSubtanglePrunerJob; import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.log.ProgressLogger; -import com.iota.iri.utils.log.interval.IntervalProgressLogger; import com.iota.iri.utils.dag.DAGHelper; import com.iota.iri.utils.dag.TraversalException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.iota.iri.utils.log.ProgressLogger; +import com.iota.iri.utils.log.interval.IntervalProgressLogger; import java.util.HashMap; import java.util.HashSet; @@ -33,6 +27,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Creates a service instance that allows us to access the business logic for {@link Snapshot}s.
*
@@ -500,7 +497,7 @@ private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot ne throws SnapshotException { try { - spentAddressesService.calculateSpentAddresses(snapshotProvider.getInitialSnapshot().getIndex(), + spentAddressesService.persistSpentAddresses(snapshotProvider.getInitialSnapshot().getIndex(), newSnapshot.getIndex()); spentAddressesProvider.writeSpentAddressesToDisk(config.getLocalSnapshotsBasePath() + diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java index 213584f711..e122701fdd 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java @@ -1,9 +1,9 @@ package com.iota.iri.service.spentaddresses; -import java.util.List; - import com.iota.iri.model.Hash; +import java.util.Collection; + /** * Find, mark and store spent addresses */ @@ -32,7 +32,7 @@ public interface SpentAddressesProvider { * @param addressHashes The addresses we want to mark * @throws SpentAddressesException If the provider fails to add an address */ - void addAddressesBatch(List addressHashes) throws SpentAddressesException; + void addAddressesBatch(Collection addressHashes) throws SpentAddressesException; /** * Writes all currently known spent addresses to disk. diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java index 14b1a33d13..5d5c1ce386 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java @@ -1,7 +1,10 @@ package com.iota.iri.service.spentaddresses; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +import java.util.Collection; + /** * * Check and calculate spent addresses @@ -18,11 +21,18 @@ public interface SpentAddressesService { boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesException; /** - * Calculate all spent addresses in between a range + * Calculates and persists all spent addresses in between a range that were validly signed * * @param fromMilestoneIndex the lower bound milestone index (inclusive) * @param toMilestoneIndex the upper bound milestone index (exclusive) * @throws Exception when anything went wrong whilst calculating. */ - void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws Exception; + void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws Exception; + + /** + * Persist all the verifiable spent from a given list of transactions + * @param transactions transactions to obtain spends from + * @throws SpentAddressesException + */ + void persistSpentAddresses(Collection transactions) throws SpentAddressesException; } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index c02cff7a63..2f3fe8d944 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -130,7 +130,7 @@ public void addAddress(Hash addressHash) throws SpentAddressesException { } @Override - public void addAddressesBatch(List addressHash) throws SpentAddressesException { + public void addAddressesBatch(Collection addressHash) throws SpentAddressesException { try { // Its bytes are always new byte[0], therefore identical in storage SpentAddress spentAddressModel = new SpentAddress(); diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java index f6fa66173b..c8a97338fd 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -14,11 +14,13 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.utils.dag.DAGHelper; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; + +import pl.touk.throwing.ThrowingPredicate; + /** * * Implementation of SpentAddressesService that calculates and checks spent addresses using the {@link Tangle} @@ -68,9 +70,9 @@ public boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesExcept } // Transaction is pending - Hash tail = findTail(hash); - if (tail != null && BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), tail).size() != 0) { - return true; + Optional tail = tailFinder.findTail(hash); + if (tail.isPresent()) { + return isBundleValid(tail.get()); } } } @@ -82,7 +84,7 @@ public boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesExcept } @Override - public void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws SpentAddressesException { + public void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws SpentAddressesException { Set addressesToCheck = new HashSet<>(); try { for (int i = fromMilestoneIndex; i < toMilestoneIndex; i++) { @@ -102,15 +104,8 @@ public void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex //Can only throw runtime exceptions in streams try { spentAddressesProvider.addAddressesBatch(addressesToCheck.stream() - .filter(address -> { - try { - return wasAddressSpentFrom(address); - } catch (SpentAddressesException e) { - throw new RuntimeException(e); - } - }) + .filter(ThrowingPredicate.unchecked(this::wasAddressSpentFrom)) .collect(Collectors.toList())); - } catch (RuntimeException e) { if (e.getCause() instanceof SpentAddressesException) { throw (SpentAddressesException) e.getCause(); @@ -120,16 +115,41 @@ public void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex } } - /** - * Walks back from the hash until a tail transaction has been found or transaction aprovee is not found. - * A tail transaction is the first transaction in a bundle, thus with index = 0 - * - * @param hash The transaction hash where we start the search from. If this is a tail, its hash is returned. - * @return The transaction hash of the tail - * @throws Exception When a model could not be loaded. - */ - private Hash findTail(Hash hash) throws Exception { - Optional optionalTail = tailFinder.findTail(hash); - return optionalTail.isPresent() ? optionalTail.get() : null; + @Override + public void persistSpentAddresses(Collection transactions) throws SpentAddressesException { + try { + Collection spentAddresses = transactions.stream() + .filter(ThrowingPredicate.unchecked(this::wasTransactionSpentFrom)) + .map(TransactionViewModel::getAddressHash).collect(Collectors.toSet()); + + spentAddressesProvider.addAddressesBatch(spentAddresses); + } catch (RuntimeException e) { + throw new SpentAddressesException("Exception while persisting spent addresses", e); + } + } + + + private boolean wasTransactionSpentFrom(TransactionViewModel tx) throws Exception { + Optional tailFromTx = tailFinder.findTailFromTx(tx); + if (tailFromTx.isPresent()) { + if (tx.value() < 0) { + // Transaction is confirmed + if (tx.snapshotIndex() != 0) { + return true; + } + + // transaction is pending + Hash tailHash = tailFromTx.get(); + return isBundleValid(tailHash); + } + } + + return false; + } + + private boolean isBundleValid(Hash tailHash) throws Exception { + List> validation = + BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), tailHash); + return (CollectionUtils.isNotEmpty(validation) && validation.get(0).get(0).getValidity() == 1); } } diff --git a/src/main/java/com/iota/iri/service/tipselection/TailFinder.java b/src/main/java/com/iota/iri/service/tipselection/TailFinder.java index e3f34120da..1d66566c8c 100644 --- a/src/main/java/com/iota/iri/service/tipselection/TailFinder.java +++ b/src/main/java/com/iota/iri/service/tipselection/TailFinder.java @@ -1,17 +1,17 @@ package com.iota.iri.service.tipselection; -import java.util.Optional; - +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; +import java.util.Optional; + /** * Finds the tail of a bundle */ -@FunctionalInterface public interface TailFinder { /** - *Method for finding a tail (current_index=0) of a bundle given any transaction in the associated bundle. + * Method for finding a tail (current_index=0) of a bundle given any transaction in the associated bundle. * * @param hash The transaction hash of any transaction in the bundle. * @return Hash of the tail transaction, or {@code Empty} if the tail is not found. @@ -19,4 +19,12 @@ public interface TailFinder { */ Optional findTail(Hash hash) throws Exception; + /** + * Method for finding a tail (current_index=0) of a bundle given any transaction in the associated bundle. + * + * @param tx any transaction in the bundle. + * @return Hash of the tail transaction, or {@code Empty} if the tail is not found. + * @throws Exception If DB fails to retrieve transactions + */ + Optional findTailFromTx(TransactionViewModel tx) throws Exception; } diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java b/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java index 52c8ee0064..e211d6c1eb 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java @@ -27,6 +27,10 @@ public TailFinderImpl(Tangle tangle) { @Override public Optional findTail(Hash hash) throws Exception { TransactionViewModel tx = TransactionViewModel.fromHash(tangle, hash); + return findTailFromTx(tx); + } + + public Optional findTailFromTx(TransactionViewModel tx) throws Exception { final Hash bundleHash = tx.getBundleHash(); long index = tx.getCurrentIndex(); while (index-- > 0 && bundleHash.equals(tx.getBundleHash())) { diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java index 3eea85d2ff..7804139f31 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java @@ -7,10 +7,17 @@ import com.iota.iri.service.snapshot.SnapshotProvider; import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; import com.iota.iri.service.tipselection.RatingCalculator; +import com.iota.iri.service.tipselection.TailFinder; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.collections.interfaces.UnIterableMap; import com.iota.iri.zmq.MessageQ; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Random; + import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -20,12 +27,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Random; - -import static com.iota.iri.controllers.TransactionViewModelTest.*; +import static com.iota.iri.controllers.TransactionViewModelTest.getRandomTransactionHash; +import static com.iota.iri.controllers.TransactionViewModelTest.getRandomTransactionTrits; +import static com.iota.iri.controllers.TransactionViewModelTest.getRandomTransactionWithTrunkAndBranch; public class WalkerAlphaTest { private static final TemporaryFolder dbFolder = new TemporaryFolder(); @@ -54,7 +58,10 @@ public static void setUp() throws Exception { tangle.init(); MessageQ messageQ = Mockito.mock(MessageQ.class); - walker = new WalkerAlpha((Optional::of), tangle, messageQ, new Random(1), new MainnetConfig()); + TailFinder tailFinder = Mockito.mock(TailFinder.class); + Mockito.when(tailFinder.findTail(Mockito.any(Hash.class))) + .then(args -> Optional.of(args.getArgumentAt(0, Hash.class))); + walker = new WalkerAlpha(tailFinder, tangle, messageQ, new Random(1), new MainnetConfig()); } From 22ec59921e272d5484d66644b1bab42b74c91a0c Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 6 Jan 2019 20:21:50 +0200 Subject: [PATCH 26/63] TransactionPruner: persist addresses upon pruning --- src/main/java/com/iota/iri/Iota.java | 3 ++- .../TransactionPrunerJob.java | 9 +++++++ .../async/AsyncTransactionPruner.java | 21 +++++++++++---- .../jobs/MilestonePrunerJob.java | 9 ++++++- .../jobs/UnconfirmedSubtanglePrunerJob.java | 27 ++++++++++++++----- 5 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index cd7996b5f6..d5a3795099 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -210,7 +210,8 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx milestoneSolidifier.init(snapshotProvider, transactionValidator); ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService); if (transactionPruner != null) { - transactionPruner.init(tangle, snapshotProvider, tipsViewModel, configuration).restoreState(); + transactionPruner.init(tangle, snapshotProvider, spentAddressesService, tipsViewModel, configuration) + .restoreState(); } transactionRequesterWorker.init(tangle, transactionRequester, tipsViewModel, node); } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java index 8ffc17cfc4..5398fcf4a5 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java @@ -2,6 +2,7 @@ import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.service.snapshot.Snapshot; +import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.storage.Tangle; /** @@ -28,6 +29,14 @@ public interface TransactionPrunerJob { */ TransactionPruner getTransactionPruner(); + /** + * Allows to set the {@link SpentAddressesService} that will ensure pruned valid transactions will have their + * spent addresses persisted the node + * + * @param spentAddressesService service to be injected + */ + void setSpentAddressesService(SpentAddressesService spentAddressesService); + /** * Allows to set the {@link Tangle} object that this job should work on. * diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index a99a9af6c6..dd7784b844 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -3,6 +3,7 @@ import com.iota.iri.conf.SnapshotConfig; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.service.snapshot.SnapshotProvider; +import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPruningException; @@ -11,15 +12,17 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.utils.thread.ThreadIdentifier; import com.iota.iri.utils.thread.ThreadUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.*; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Creates a {@link TransactionPruner} that is able to process it's jobs asynchronously in the background and persists * its state in a file on the hard disk of the node.
@@ -59,6 +62,11 @@ public class AsyncTransactionPruner implements TransactionPruner { */ private SnapshotProvider snapshotProvider; + /** + * Used to check whether an address was spent from before it is pruned. + */ + private SpentAddressesService spentAddressesService; + /** * Manager for the tips (required for removing pruned transactions from this manager). */ @@ -119,11 +127,13 @@ public class AsyncTransactionPruner implements TransactionPruner { * @param config Configuration with important snapshot related configuration parameters * @return the initialized instance itself to allow chaining */ - public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvider, TipsViewModel tipsViewModel, - SnapshotConfig config) { + public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvider, + SpentAddressesService spentAddressesService, TipsViewModel tipsViewModel, + SnapshotConfig config) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; + this.spentAddressesService = spentAddressesService; this.tipsViewModel = tipsViewModel; this.config = config; @@ -144,6 +154,7 @@ public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvi @Override public void addJob(TransactionPrunerJob job) throws TransactionPruningException { job.setTransactionPruner(this); + job.setSpentAddressesService(spentAddressesService); job.setTangle(tangle); job.setTipsViewModel(tipsViewModel); job.setSnapshot(snapshotProvider.getInitialSnapshot()); diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index 4da141d963..459dd7e412 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -5,6 +5,7 @@ import com.iota.iri.model.IntegerIndex; import com.iota.iri.model.persistables.Milestone; import com.iota.iri.model.persistables.Transaction; +import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.storage.Indexable; @@ -12,7 +13,8 @@ import com.iota.iri.utils.Pair; import com.iota.iri.utils.dag.DAGHelper; -import java.util.*; +import java.util.ArrayList; +import java.util.List; /** * Represents a cleanup job for {@link com.iota.iri.service.transactionpruning.TransactionPruner}s that removes @@ -97,6 +99,11 @@ private MilestonePrunerJob(int startingIndex, int currentIndex, int targetIndex) } } + @Override + public void setSpentAddressesService(SpentAddressesService spentAddressesService) { + //Does nothing. Confirmed transactions spent are persisted during the local snapshot. + } + /** * {@inheritDoc} * diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java index 9189e96a7f..4496a57c07 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java @@ -1,8 +1,10 @@ package com.iota.iri.service.transactionpruning.jobs; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; import com.iota.iri.model.persistables.Transaction; +import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.storage.Indexable; @@ -10,8 +12,10 @@ import com.iota.iri.utils.Pair; import com.iota.iri.utils.dag.DAGHelper; -import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.stream.Collectors; /** * Represents a job for the {@link com.iota.iri.service.transactionpruning.TransactionPruner} that cleans up all @@ -20,6 +24,8 @@ * It is used to clean up orphaned subtangles when they become irrelevant for the ledger. */ public class UnconfirmedSubtanglePrunerJob extends AbstractTransactionPrunerJob { + + private SpentAddressesService spentAddressesService; /** * Holds the hash of the transaction that shall have its unconfirmed approvers cleaned. */ @@ -48,6 +54,11 @@ public UnconfirmedSubtanglePrunerJob(Hash transactionHash) { this.transactionHash = transactionHash; } + @Override + public void setSpentAddressesService(SpentAddressesService spentAddressesService) { + this.spentAddressesService = spentAddressesService; + } + /** * {@inheritDoc} * @@ -59,15 +70,19 @@ public void process() throws TransactionPruningException { if (getStatus() != TransactionPrunerJobStatus.DONE) { setStatus(TransactionPrunerJobStatus.RUNNING); + Collection unconfirmedTxs = new HashSet<>(); try { - List>> elementsToDelete = new ArrayList<>(); DAGHelper.get(getTangle()).traverseApprovers( transactionHash, approverTransaction -> approverTransaction.snapshotIndex() == 0, - approverTransaction -> elementsToDelete.add(new Pair<>( - approverTransaction.getHash(), Transaction.class - )) - ); + unconfirmedTxs::add + ); + + //Only persist to db + spentAddressesService.persistSpentAddresses(unconfirmedTxs); + List>> elementsToDelete = unconfirmedTxs.stream() + .map(tx -> new Pair<>((Indexable) tx.getHash(), Transaction.class)) + .collect(Collectors.toList()); // clean database entries getTangle().deleteBatch(elementsToDelete); From ca96bf9eeb37c7c06c71494629c77c04d022a679 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 6 Jan 2019 11:48:58 +0200 Subject: [PATCH 27/63] Revert "disable local-snapshots" This reverts commit 793258d --- src/main/java/com/iota/iri/Iota.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 8a9ad281a9..807d9d0c42 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -114,13 +114,17 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh // new refactored instances snapshotProvider = new SnapshotProviderImpl(); snapshotService = new SnapshotServiceImpl(); - localSnapshotManager = null; + localSnapshotManager = configuration.getLocalSnapshotsEnabled() + ? new LocalSnapshotManagerImpl() + : null; milestoneService = new MilestoneServiceImpl(); latestMilestoneTracker = new LatestMilestoneTrackerImpl(); latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl(); seenMilestonesRetriever = new SeenMilestonesRetrieverImpl(); milestoneSolidifier = new MilestoneSolidifierImpl(); - transactionPruner = null; + transactionPruner = configuration.getLocalSnapshotsEnabled() && configuration.getLocalSnapshotsPruningEnabled() + ? new AsyncTransactionPruner() + : null; transactionRequesterWorker = new TransactionRequesterWorkerImpl(); // legacy code From 7072a7d2b96d926032e0aab35ed701693807f311 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 6 Jan 2019 23:02:31 +0200 Subject: [PATCH 28/63] bump version to 1.6.0-RC13 --- DOCKER.md | 6 +++--- pom.xml | 2 +- src/main/java/com/iota/iri/IRI.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index 7a565078cc..c2a1ea10f5 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -2,13 +2,13 @@ Run the official iotaledger/iri container, passing the mandatory -p option: -```docker run iotaledger/iri:v1.6.0-RC12 -p 14265``` +```docker run iotaledger/iri:v1.6.0-RC13 -p 14265``` This will get your a running IRI with its API listening on port 14265, no neighbours and an empty database. The IRI Docker container by default expects data at /iri/data. Use the `-v` option of the `docker run` command to mount volumes so to have persistent data. You can also pass more command line options to the docker run command and those will be passed to IRI. If you want to use a iri.ini file with the docker container, supposing it's stored under /path/to/conf/iri.ini on your docker host, then pass `-v /path/to/conf:/iri/conf` and add -c /iri/conf/iri.ini as docker run arguments. So for example the `docker run` command above would become: -```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC12 -p 14265 -c /iri/conf/iri.ini``` +```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC13 -p 14265 -c /iri/conf/iri.ini``` Please refer to the IRI documentation for further command line options and iri.ini options. @@ -61,7 +61,7 @@ ExecStart=/usr/bin/docker run \ -p 14265:14265 \ -p 15600:15600 \ -p 14600:14600/udp \ -iotaledger/iri:v1.6.0-RC12 \ +iotaledger/iri:v1.6.0-RC13 \ -p 14265 \ --zmq-enabled \ --testnet diff --git a/pom.xml b/pom.xml index fd7e5f3785..5b87aefa97 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.iota iri - 1.6.0-RC12 + 1.6.0-RC13 IRI IOTA Reference Implementation diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index 1aebd7a09e..3bdd57cbf9 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -42,7 +42,7 @@ public class IRI { public static final String MAINNET_NAME = "IRI"; public static final String TESTNET_NAME = "IRI Testnet"; - public static final String VERSION = "1.6.0-RC12"; + public static final String VERSION = "1.6.0-RC13"; /** * The entry point of IRI. From c7c3d1ec0ed9d2b211936b48601b64182fc8b36e Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 6 Jan 2019 23:34:38 +0200 Subject: [PATCH 29/63] fixed according to codacy comments --- .../impl/SpentAddressesServiceImpl.java | 19 ++++++++----------- .../tipselection/impl/TailFinderImpl.java | 1 + 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java index c8a97338fd..a444a578a1 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -128,20 +128,17 @@ public void persistSpentAddresses(Collection transactions) } } - private boolean wasTransactionSpentFrom(TransactionViewModel tx) throws Exception { Optional tailFromTx = tailFinder.findTailFromTx(tx); - if (tailFromTx.isPresent()) { - if (tx.value() < 0) { - // Transaction is confirmed - if (tx.snapshotIndex() != 0) { - return true; - } - - // transaction is pending - Hash tailHash = tailFromTx.get(); - return isBundleValid(tailHash); + if (tailFromTx.isPresent() && tx.value() < 0) { + // Transaction is confirmed + if (tx.snapshotIndex() != 0) { + return true; } + + // transaction is pending + Hash tailHash = tailFromTx.get(); + return isBundleValid(tailHash); } return false; diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java b/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java index e211d6c1eb..244a1da0c0 100644 --- a/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java +++ b/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java @@ -30,6 +30,7 @@ public Optional findTail(Hash hash) throws Exception { return findTailFromTx(tx); } + @Override public Optional findTailFromTx(TransactionViewModel tx) throws Exception { final Hash bundleHash = tx.getBundleHash(); long index = tx.getCurrentIndex(); From 9def5e96e5764efcd76b595ad0ca082c20ee98a2 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Mon, 7 Jan 2019 20:02:40 +0200 Subject: [PATCH 30/63] bring back initialization of services --- src/main/java/com/iota/iri/Iota.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index bc1864f101..c41dcaf6cb 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -198,7 +198,9 @@ public void init() throws Exception { } } - private void injectDependencies() throws SnapshotException, TransactionPruningException { + private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException { + spentAddressesProvider.init(tangle, configuration); + spentAddressesService.init(tangle, snapshotProvider, spentAddressesProvider); snapshotProvider.init(configuration); snapshotService.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, configuration); if (localSnapshotManager != null) { From 86640aea9425b0947530616fb643b7030c65f427 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Mon, 7 Jan 2019 20:39:21 +0200 Subject: [PATCH 31/63] While pruning spending confirmed transactions, make sure they are recorded as spent --- .../jobs/AbstractTransactionPrunerJob.java | 11 +++++++++ .../jobs/MilestonePrunerJob.java | 23 +++++++++---------- .../jobs/UnconfirmedSubtanglePrunerJob.java | 6 ----- .../com/iota/iri/utils/dag/DAGHelper.java | 7 ++++-- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java index 4636129a2e..ebaf0744df 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/AbstractTransactionPrunerJob.java @@ -2,6 +2,7 @@ import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.service.snapshot.Snapshot; +import com.iota.iri.service.spentaddresses.SpentAddressesService; import com.iota.iri.service.transactionpruning.TransactionPruner; import com.iota.iri.service.transactionpruning.TransactionPrunerJob; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; @@ -22,6 +23,11 @@ public abstract class AbstractTransactionPrunerJob implements TransactionPrunerJ */ private TransactionPruner transactionPruner; + /** + * Ascertains that pruned transactions are recorded as spent addresses where necessary + */ + protected SpentAddressesService spentAddressesService; + /** * Holds a reference to the tangle object which acts as a database interface. */ @@ -85,6 +91,11 @@ public TipsViewModel getTipsViewModel() { return tipsViewModel; } + @Override + public void setSpentAddressesService(SpentAddressesService spentAddressesService) { + this.spentAddressesService = spentAddressesService; + } + /** * {@inheritDoc} */ diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index 459dd7e412..8974c6a981 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -5,7 +5,7 @@ import com.iota.iri.model.IntegerIndex; import com.iota.iri.model.persistables.Milestone; import com.iota.iri.model.persistables.Transaction; -import com.iota.iri.service.spentaddresses.SpentAddressesService; +import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.storage.Indexable; @@ -99,11 +99,6 @@ private MilestonePrunerJob(int startingIndex, int currentIndex, int targetIndex) } } - @Override - public void setSpentAddressesService(SpentAddressesService spentAddressesService) { - //Does nothing. Confirmed transactions spent are persisted during the local snapshot. - } - /** * {@inheritDoc} * @@ -264,13 +259,17 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { elementsToDelete.add(new Pair<>(milestoneViewModel.getHash(), Transaction.class)); elementsToDelete.add(new Pair<>(new IntegerIndex(milestoneViewModel.index()), Milestone.class)); - DAGHelper.get(getTangle()).traverseApprovees( - milestoneViewModel.getHash(), + DAGHelper.get(getTangle()).traverseApprovees(milestoneViewModel.getHash(), approvedTransaction -> approvedTransaction.snapshotIndex() >= milestoneViewModel.index(), - approvedTransaction -> elementsToDelete.add( - new Pair<>(approvedTransaction.getHash(), Transaction.class) - ) - ); + approvedTransaction -> { + if (approvedTransaction.value() < 0 && + !spentAddressesService.wasAddressSpentFrom(approvedTransaction.getAddressHash())) { + throw new SpentAddressesException( + "Pruned spend transaction did not have its spent address recorded"); + } else { + elementsToDelete.add(new Pair<>(approvedTransaction.getHash(), Transaction.class)); + } + }); } return elementsToDelete; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java index 4496a57c07..780d078267 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/UnconfirmedSubtanglePrunerJob.java @@ -25,7 +25,6 @@ */ public class UnconfirmedSubtanglePrunerJob extends AbstractTransactionPrunerJob { - private SpentAddressesService spentAddressesService; /** * Holds the hash of the transaction that shall have its unconfirmed approvers cleaned. */ @@ -54,11 +53,6 @@ public UnconfirmedSubtanglePrunerJob(Hash transactionHash) { this.transactionHash = transactionHash; } - @Override - public void setSpentAddressesService(SpentAddressesService spentAddressesService) { - this.spentAddressesService = spentAddressesService; - } - /** * {@inheritDoc} * diff --git a/src/main/java/com/iota/iri/utils/dag/DAGHelper.java b/src/main/java/com/iota/iri/utils/dag/DAGHelper.java index ec814daf1b..4a3f72b0c3 100644 --- a/src/main/java/com/iota/iri/utils/dag/DAGHelper.java +++ b/src/main/java/com/iota/iri/utils/dag/DAGHelper.java @@ -9,6 +9,8 @@ import java.util.function.Consumer; import java.util.function.Predicate; +import pl.touk.throwing.ThrowingConsumer; + /** * This class offers generic functions for recurring tasks that are related to the tangle and that otherwise would have * to be implemented over and over again in different parts of the code. @@ -161,7 +163,7 @@ public void traverseApprovers(Hash startingTransactionHash, */ public void traverseApprovees(Hash startingTransactionHash, Predicate condition, - Consumer currentTransactionConsumer, + ThrowingConsumer currentTransactionConsumer, Set processedTransactions) throws TraversalException { Queue transactionsToExamine = new ArrayDeque<>(Collections.singleton(startingTransactionHash)); try { @@ -205,7 +207,8 @@ public void traverseApprovees(Hash startingTransactionHash, */ public void traverseApprovees(Hash startingTransactionHash, Predicate condition, - Consumer currentTransactionConsumer) throws TraversalException { + ThrowingConsumer currentTransactionConsumer) + throws TraversalException { traverseApprovees(startingTransactionHash, condition, currentTransactionConsumer, new HashSet<>()); } From d801de5e91a293babb14439052f9488310a62a6c Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Mon, 7 Jan 2019 20:48:14 +0200 Subject: [PATCH 32/63] Use tailFinder more efficiently --- .../service/spentaddresses/impl/SpentAddressesServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java index a444a578a1..15605e2c79 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -70,7 +70,7 @@ public boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesExcept } // Transaction is pending - Optional tail = tailFinder.findTail(hash); + Optional tail = tailFinder.findTailFromTx(tx); if (tail.isPresent()) { return isBundleValid(tail.get()); } From f8eef61e8f2a51e19308696f7e90b36bedd4fbf4 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Tue, 8 Jan 2019 21:55:27 +0200 Subject: [PATCH 33/63] bump version to 1.6.0-RC14 --- DOCKER.md | 6 +++--- pom.xml | 2 +- src/main/java/com/iota/iri/IRI.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index c2a1ea10f5..65c66b5af8 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -2,13 +2,13 @@ Run the official iotaledger/iri container, passing the mandatory -p option: -```docker run iotaledger/iri:v1.6.0-RC13 -p 14265``` +```docker run iotaledger/iri:v1.6.0-RC14 -p 14265``` This will get your a running IRI with its API listening on port 14265, no neighbours and an empty database. The IRI Docker container by default expects data at /iri/data. Use the `-v` option of the `docker run` command to mount volumes so to have persistent data. You can also pass more command line options to the docker run command and those will be passed to IRI. If you want to use a iri.ini file with the docker container, supposing it's stored under /path/to/conf/iri.ini on your docker host, then pass `-v /path/to/conf:/iri/conf` and add -c /iri/conf/iri.ini as docker run arguments. So for example the `docker run` command above would become: -```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC13 -p 14265 -c /iri/conf/iri.ini``` +```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC14 -p 14265 -c /iri/conf/iri.ini``` Please refer to the IRI documentation for further command line options and iri.ini options. @@ -61,7 +61,7 @@ ExecStart=/usr/bin/docker run \ -p 14265:14265 \ -p 15600:15600 \ -p 14600:14600/udp \ -iotaledger/iri:v1.6.0-RC13 \ +iotaledger/iri:v1.6.0-RC14 \ -p 14265 \ --zmq-enabled \ --testnet diff --git a/pom.xml b/pom.xml index dda1ddb068..9efe4c834f 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.iota iri - 1.6.0-RC13 + 1.6.0-RC14 IRI IOTA Reference Implementation diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index 6c6380a38e..27b0de238c 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -42,7 +42,7 @@ public class IRI { public static final String MAINNET_NAME = "IRI"; public static final String TESTNET_NAME = "IRI Testnet"; - public static final String VERSION = "1.6.0-RC13"; + public static final String VERSION = "1.6.0-RC14"; /** * The entry point of IRI. From aae51a2a14bba6fc19d239f341a632b029f049bb Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Mon, 7 Jan 2019 17:21:00 +0200 Subject: [PATCH 34/63] rocksdb modification --- src/main/java/com/iota/iri/Iota.java | 19 ++- .../rocksDB/RocksDBPersistenceProvider.java | 108 +++++++----------- 2 files changed, 61 insertions(+), 66 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index c41dcaf6cb..adfd51d3fa 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -4,6 +4,8 @@ import com.iota.iri.conf.TipSelConfig; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.StateDiff; +import com.iota.iri.model.persistables.*; import com.iota.iri.network.Node; import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.UDPReceiver; @@ -28,10 +30,13 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.storage.ZmqPublishProvider; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; +import com.iota.iri.utils.IotaUtils; import com.iota.iri.utils.Pair; import com.iota.iri.zmq.MessageQ; import java.security.SecureRandom; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import org.apache.commons.lang3.NotImplementedException; @@ -283,7 +288,19 @@ private void initializeTangle() { tangle.addPersistenceProvider(new RocksDBPersistenceProvider( configuration.getDbPath(), configuration.getDbLogPath(), - configuration.getDbCacheSize())); + configuration.getDbCacheSize(), + new HashMap>() {{ + put("transaction", Transaction.class); + put("milestone", Milestone.class); + put("stateDiff", StateDiff.class); + put("address", Address.class); + put("approvee", Approvee.class); + put("bundle", Bundle.class); + put("obsoleteTag", ObsoleteTag.class); + put("tag", Tag.class); + }}, + new Pair<>("transaction-metadata", Transaction.class)) + ); break; } default: { diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java index 3b3cc70b46..8b644a07d7 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java @@ -1,8 +1,6 @@ package com.iota.iri.storage.rocksDB; import com.iota.iri.model.HashFactory; -import com.iota.iri.model.StateDiff; -import com.iota.iri.model.persistables.*; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.PersistenceProvider; @@ -16,6 +14,7 @@ import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.SystemUtils; import org.rocksdb.*; import org.rocksdb.util.SizeUnit; @@ -29,40 +28,17 @@ public class RocksDBPersistenceProvider implements PersistenceProvider { private static final Pair PAIR_OF_NULLS = new Pair<>(null, null); - private final List columnFamilyNames = Arrays.asList( - new String(RocksDB.DEFAULT_COLUMN_FAMILY), - "spentAddress", - "transaction", - "transaction-metadata", - "milestone", - "stateDiff", - "address", - "approvee", - "bundle", - "obsoleteTag", - "tag" - ); - private final List columnFamilyHandles = new ArrayList<>(); private final SecureRandom seed = new SecureRandom(); private final String dbPath; private final String logPath; private final int cacheSize; - - private ColumnFamilyHandle spentAddressHandle; - private ColumnFamilyHandle transactionHandle; - private ColumnFamilyHandle transactionMetadataHandle; - private ColumnFamilyHandle milestoneHandle; - private ColumnFamilyHandle stateDiffHandle; - private ColumnFamilyHandle addressHandle; - private ColumnFamilyHandle approveeHandle; - private ColumnFamilyHandle bundleHandle; - private ColumnFamilyHandle obsoleteTagHandle; - private ColumnFamilyHandle tagHandle; + private final Map> columnFamilies; + private final Pair> metadataColumnFamily; private Map, ColumnFamilyHandle> classTreeMap; - private Map, ColumnFamilyHandle> metadataReference; + private Map, ColumnFamilyHandle> metadataReference = Collections.emptyMap(); private RocksDB db; // DBOptions is only used in initDB(). However, it is closeable - so we keep a reference for shutdown. @@ -70,17 +46,21 @@ public class RocksDBPersistenceProvider implements PersistenceProvider { private BloomFilter bloomFilter; private boolean available; - public RocksDBPersistenceProvider(String dbPath, String logPath, int cacheSize) { + public RocksDBPersistenceProvider(String dbPath, String logPath, int cacheSize, + Map> columnFamilies, + Pair> metadataColumnFamily) { this.dbPath = dbPath; this.logPath = logPath; this.cacheSize = cacheSize; + this.columnFamilies = columnFamilies; + this.metadataColumnFamily = metadataColumnFamily; + } @Override public void init() { log.info("Initializing Database Backend... "); - initDB(dbPath, logPath); - initClassTreeMap(); + initDB(dbPath, logPath, columnFamilies); available = true; log.info("RocksDB persistence provider initialized."); } @@ -90,23 +70,6 @@ public boolean isAvailable() { return this.available; } - private void initClassTreeMap() { - Map, ColumnFamilyHandle> classMap = new LinkedHashMap<>(); - classMap.put(SpentAddress.class, spentAddressHandle); - classMap.put(Transaction.class, transactionHandle); - classMap.put(Milestone.class, milestoneHandle); - classMap.put(StateDiff.class, stateDiffHandle); - classMap.put(Address.class, addressHandle); - classMap.put(Approvee.class, approveeHandle); - classMap.put(Bundle.class, bundleHandle); - classMap.put(ObsoleteTag.class, obsoleteTagHandle); - classMap.put(Tag.class, tagHandle); - classTreeMap = classMap; - - Map, ColumnFamilyHandle> metadataHashMap = new HashMap<>(); - metadataHashMap.put(Transaction.class, transactionMetadataHandle); - metadataReference = metadataHashMap; - } @Override public void shutdown() { @@ -428,10 +391,10 @@ public void restoreBackup(String path, String logPath) throws Exception { backupEngine.restoreDbFromLatestBackup(path, logPath, restoreOptions); } } - initDB(path, logPath); + initDB(path, logPath, columnFamilies); } - private void initDB(String path, String logPath) { + private void initDB(String path, String logPath, Map> columnFamilies) { try { try { RocksDB.loadLibrary(); @@ -490,14 +453,23 @@ private void initDB(String path, String logPath) { .setWriteBufferSize(2 * SizeUnit.MB); List columnFamilyDescriptors = new ArrayList<>(); - for (String name : columnFamilyNames) { + //Add default column family. Main motivation is to not change legacy code + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions)); + for (String name : columnFamilies.keySet()) { columnFamilyDescriptors.add(new ColumnFamilyDescriptor(name.getBytes(), columnFamilyOptions)); } + // metadata descriptor is always last + if (metadataColumnFamily != null) { + columnFamilyDescriptors.add( + new ColumnFamilyDescriptor(metadataColumnFamily.low.getBytes(), columnFamilyOptions)); + metadataReference = new HashMap<>(); + } + db = RocksDB.open(options, path, columnFamilyDescriptors, columnFamilyHandles); db.enableFileDeletions(true); - fillModelColumnHandles(); + initClassTreeMap(columnFamilyDescriptors); } catch (Exception e) { log.error("Error while initializing RocksDb", e); @@ -505,22 +477,28 @@ private void initDB(String path, String logPath) { } } - private void fillModelColumnHandles() throws Exception { - int i = 0; - spentAddressHandle = columnFamilyHandles.get(++i); - transactionHandle = columnFamilyHandles.get(++i); - transactionMetadataHandle = columnFamilyHandles.get(++i); - milestoneHandle = columnFamilyHandles.get(++i); - stateDiffHandle = columnFamilyHandles.get(++i); - addressHandle = columnFamilyHandles.get(++i); - approveeHandle = columnFamilyHandles.get(++i); - bundleHandle = columnFamilyHandles.get(++i); - obsoleteTagHandle = columnFamilyHandles.get(++i); - tagHandle = columnFamilyHandles.get(++i); - + private void initClassTreeMap(List columnFamilyDescriptors) throws Exception { + Map, ColumnFamilyHandle> classMap = new LinkedHashMap<>(); + String mcfName = metadataColumnFamily == null ? "" : metadataColumnFamily.low; + //skip default column + int i = 1; + for (; i < columnFamilyDescriptors.size(); i++) { + + String name = new String(columnFamilyDescriptors.get(i).columnFamilyName()); + if (name.equals(mcfName)) { + Map, ColumnFamilyHandle> metadataRef = new HashMap<>(); + metadataRef.put(metadataColumnFamily.hi, columnFamilyHandles.get(i)); + metadataReference = MapUtils.unmodifiableMap(metadataRef); + } + else { + classMap.put(columnFamilies.get(name), columnFamilyHandles.get(i)); + } + } for (; ++i < columnFamilyHandles.size(); ) { db.dropColumnFamily(columnFamilyHandles.get(i)); } + + classTreeMap = classMap; } // 2018 March 28 - Unused Code From 831b0e6fdfa12c2cc321899f59db871fdd1a9b64 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Tue, 8 Jan 2019 17:39:04 +0200 Subject: [PATCH 35/63] rocksdb initialization --- src/main/java/com/iota/iri/Iota.java | 28 ++++++++++--------- .../rocksDB/RocksDBPersistenceProvider.java | 2 +- .../com/iota/iri/BundleValidatorTest.java | 5 +++- .../iota/iri/TransactionValidatorTest.java | 2 +- .../dbbenchmark/states/DbState.java | 5 ++-- .../iri/controllers/BundleViewModelTest.java | 6 ++-- .../controllers/MilestoneViewModelTest.java | 6 ++-- .../controllers/TransactionViewModelTest.java | 6 ++-- .../impl/CumulativeWeightCalculatorTest.java | 5 ++-- .../tipselection/impl/RatingOneTest.java | 6 ++-- .../tipselection/impl/TailFinderImplTest.java | 6 ++-- .../impl/WalkValidatorImplTest.java | 6 ++-- .../tipselection/impl/WalkerAlphaTest.java | 6 ++-- .../java/com/iota/iri/storage/TangleTest.java | 6 ++-- .../RocksDBPersistenceProviderTest.java | 4 ++- 15 files changed, 62 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index adfd51d3fa..a09f0acaba 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -30,13 +30,12 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.storage.ZmqPublishProvider; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; -import com.iota.iri.utils.IotaUtils; import com.iota.iri.utils.Pair; import com.iota.iri.zmq.MessageQ; import java.security.SecureRandom; -import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import org.apache.commons.lang3.NotImplementedException; @@ -76,6 +75,18 @@ */ public class Iota { private static final Logger log = LoggerFactory.getLogger(Iota.class); + public static final HashMap> COLUMN_FAMILIES = + new LinkedHashMap>() {{ + put("transaction", Transaction.class); + put("milestone", Milestone.class); + put("stateDiff", StateDiff.class); + put("address", Address.class); + put("approvee", Approvee.class); + put("bundle", Bundle.class); + put("obsoleteTag", ObsoleteTag.class); + put("tag", Tag.class); + }}; + public static final Pair> METADATA_COLUMN_FAMILY = new Pair<>("transaction-metadata", Transaction.class); public final SpentAddressesProviderImpl spentAddressesProvider; @@ -289,17 +300,8 @@ private void initializeTangle() { configuration.getDbPath(), configuration.getDbLogPath(), configuration.getDbCacheSize(), - new HashMap>() {{ - put("transaction", Transaction.class); - put("milestone", Milestone.class); - put("stateDiff", StateDiff.class); - put("address", Address.class); - put("approvee", Approvee.class); - put("bundle", Bundle.class); - put("obsoleteTag", ObsoleteTag.class); - put("tag", Tag.class); - }}, - new Pair<>("transaction-metadata", Transaction.class)) + COLUMN_FAMILIES, + METADATA_COLUMN_FAMILY) ); break; } diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java index 8b644a07d7..93c91e7239 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java @@ -498,7 +498,7 @@ private void initClassTreeMap(List columnFamilyDescripto db.dropColumnFamily(columnFamilyHandles.get(i)); } - classTreeMap = classMap; + classTreeMap = MapUtils.unmodifiableMap(classMap); } // 2018 March 28 - Unused Code diff --git a/src/test/java/com/iota/iri/BundleValidatorTest.java b/src/test/java/com/iota/iri/BundleValidatorTest.java index b6f2387f0e..a3a09444a4 100644 --- a/src/test/java/com/iota/iri/BundleValidatorTest.java +++ b/src/test/java/com/iota/iri/BundleValidatorTest.java @@ -28,7 +28,10 @@ public static void setUp() throws Exception { TemporaryFolder logFolder = new TemporaryFolder(); dbFolder.create(); logFolder.create(); - tangle.addPersistenceProvider(new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000)); + tangle.addPersistenceProvider( + new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), + logFolder.getRoot().getAbsolutePath(), 1000, Iota.COLUMN_FAMILIES, + Iota.METADATA_COLUMN_FAMILY)); tangle.init(); snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); } diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java index e64162ffbc..44b6734568 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/TransactionValidatorTest.java @@ -39,7 +39,7 @@ public static void setUp() throws Exception { snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); tangle.addPersistenceProvider( new RocksDBPersistenceProvider( - dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000)); + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); tangle.init(); TipsViewModel tipsViewModel = new TipsViewModel(); MessageQ messageQ = Mockito.mock(MessageQ.class); diff --git a/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java b/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java index a604c98b02..26e6231417 100644 --- a/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java +++ b/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java @@ -1,5 +1,6 @@ package com.iota.iri.benchmarks.dbbenchmark.states; +import com.iota.iri.Iota; import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.BaseIotaConfig; import com.iota.iri.conf.MainnetConfig; @@ -41,8 +42,8 @@ public void setup() throws Exception { + dbFolder.getAbsolutePath()); } logFolder.mkdirs(); - PersistenceProvider dbProvider = new RocksDBPersistenceProvider(dbFolder.getPath(), logFolder.getPath(), - BaseIotaConfig.Defaults.DB_CACHE_SIZE); + PersistenceProvider dbProvider = new RocksDBPersistenceProvider( + dbFolder.getAbsolutePath(), logFolder.getAbsolutePath(), BaseIotaConfig.Defaults.DB_CACHE_SIZE, Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); dbProvider.init(); tangle = new Tangle(); snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); diff --git a/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java b/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java index b8dca71094..b838a09542 100644 --- a/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java @@ -1,5 +1,6 @@ package com.iota.iri.controllers; +import com.iota.iri.Iota; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import org.junit.After; @@ -20,8 +21,9 @@ public void setUp() throws Exception { dbFolder.create(); logFolder.create(); RocksDBPersistenceProvider rocksDBPersistenceProvider; - rocksDBPersistenceProvider = new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), - logFolder.getRoot().getAbsolutePath(),1000); + rocksDBPersistenceProvider = new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, + Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); tangle.addPersistenceProvider(rocksDBPersistenceProvider); tangle.init(); diff --git a/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java b/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java index 354796f087..e602c77553 100644 --- a/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java @@ -1,5 +1,6 @@ package com.iota.iri.controllers; +import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; @@ -24,8 +25,9 @@ public void setUpTest() throws Exception { dbFolder.create(); logFolder.create(); RocksDBPersistenceProvider rocksDBPersistenceProvider; - rocksDBPersistenceProvider = new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), - logFolder.getRoot().getAbsolutePath(),1000); + rocksDBPersistenceProvider = new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, + Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); tangle.addPersistenceProvider(rocksDBPersistenceProvider); tangle.init(); } diff --git a/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java b/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java index b3ae7f91c6..fddc1863f8 100644 --- a/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java @@ -1,5 +1,6 @@ package com.iota.iri.controllers; +import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.crypto.SpongeFactory; import com.iota.iri.model.Hash; @@ -41,8 +42,9 @@ public static void setUp() throws Exception { dbFolder.create(); logFolder.create(); RocksDBPersistenceProvider rocksDBPersistenceProvider; - rocksDBPersistenceProvider = new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), - logFolder.getRoot().getAbsolutePath(),1000); + rocksDBPersistenceProvider = new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, + Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); tangle.addPersistenceProvider(rocksDBPersistenceProvider); tangle.init(); snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java index 94bc7171c1..ade3ea34ce 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java @@ -1,6 +1,7 @@ package com.iota.iri.service.tipselection.impl; +import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.ApproveeViewModel; import com.iota.iri.controllers.TransactionViewModel; @@ -48,8 +49,8 @@ public static void setUp() throws Exception { snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); dbFolder.create(); logFolder.create(); - tangle.addPersistenceProvider(new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), logFolder - .getRoot().getAbsolutePath(), 1000)); + tangle.addPersistenceProvider( new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); tangle.init(); cumulativeWeightCalculator = new CumulativeWeightCalculator(tangle, snapshotProvider); } diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java index 2c12f945be..45777f3322 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java @@ -1,5 +1,6 @@ package com.iota.iri.service.tipselection.impl; +import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.HashId; @@ -39,8 +40,9 @@ public static void setUp() throws Exception { snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); dbFolder.create(); logFolder.create(); - tangle.addPersistenceProvider(new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), logFolder - .getRoot().getAbsolutePath(), 1000)); + tangle.addPersistenceProvider( new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, + Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); tangle.init(); rating = new RatingOne(tangle); } diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java index 281e8fe710..ad070fe292 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java @@ -1,5 +1,6 @@ package com.iota.iri.service.tipselection.impl; +import com.iota.iri.Iota; import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.TransactionViewModel; @@ -43,8 +44,9 @@ public static void setUp() throws Exception { snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); dbFolder.create(); logFolder.create(); - tangle.addPersistenceProvider(new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), logFolder - .getRoot().getAbsolutePath(), 1000)); + tangle.addPersistenceProvider( new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, + Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); tangle.init(); } diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java index 082a76e3a6..14a6f7ffaf 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java @@ -1,5 +1,6 @@ package com.iota.iri.service.tipselection.impl; +import com.iota.iri.Iota; import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.conf.TipSelConfig; @@ -49,8 +50,9 @@ public static void setUp() throws Exception { snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); dbFolder.create(); logFolder.create(); - tangle.addPersistenceProvider(new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), logFolder - .getRoot().getAbsolutePath(), 1000)); + tangle.addPersistenceProvider( new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, + Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); tangle.init(); } diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java index 7804139f31..8741f6e59e 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java @@ -1,5 +1,6 @@ package com.iota.iri.service.tipselection.impl; +import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; @@ -53,8 +54,9 @@ public static void setUp() throws Exception { snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); dbFolder.create(); logFolder.create(); - tangle.addPersistenceProvider(new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), logFolder - .getRoot().getAbsolutePath(), 1000)); + tangle.addPersistenceProvider( new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, + Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); tangle.init(); MessageQ messageQ = Mockito.mock(MessageQ.class); diff --git a/src/test/java/com/iota/iri/storage/TangleTest.java b/src/test/java/com/iota/iri/storage/TangleTest.java index 26ff768a6e..d10f57225b 100644 --- a/src/test/java/com/iota/iri/storage/TangleTest.java +++ b/src/test/java/com/iota/iri/storage/TangleTest.java @@ -1,5 +1,6 @@ package com.iota.iri.storage; +import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.crypto.SpongeFactory; @@ -31,8 +32,9 @@ public void setUp() throws Exception { dbFolder.create(); logFolder.create(); RocksDBPersistenceProvider rocksDBPersistenceProvider; - rocksDBPersistenceProvider = new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), - logFolder.getRoot().getAbsolutePath(),1000); + rocksDBPersistenceProvider = new RocksDBPersistenceProvider( + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, + Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); tangle.addPersistenceProvider(rocksDBPersistenceProvider); tangle.init(); snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); diff --git a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java index c2ee8f28a3..dd87770a22 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java @@ -1,5 +1,6 @@ package com.iota.iri.storage.rocksDB; +import com.iota.iri.Iota; import com.iota.iri.model.IntegerIndex; import com.iota.iri.model.persistables.Transaction; import com.iota.iri.storage.Indexable; @@ -23,7 +24,8 @@ public class RocksDBPersistenceProviderTest { @BeforeClass public static void setUpDb() { - rocksDBPersistenceProvider = new RocksDBPersistenceProvider(dbPath, dbLogPath, 10000); + rocksDBPersistenceProvider = new RocksDBPersistenceProvider( + dbPath, dbLogPath,1000, Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); rocksDBPersistenceProvider.init(); } From 3ced3c90e6a39d43fb1da1e350b718c4ac01feb7 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Tue, 8 Jan 2019 21:35:53 +0200 Subject: [PATCH 36/63] use rocksdb in snapshot provider --- src/main/java/com/iota/iri/Iota.java | 2 +- .../impl/SpentAddressesProviderImpl.java | 35 +++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index a09f0acaba..59ec4b1c9f 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -215,7 +215,7 @@ public void init() throws Exception { } private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException { - spentAddressesProvider.init(tangle, configuration); + spentAddressesProvider.init(configuration); spentAddressesService.init(tangle, snapshotProvider, spentAddressesProvider); snapshotProvider.init(configuration); snapshotService.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, configuration); diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 2f3fe8d944..6cd10c4972 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -10,19 +10,19 @@ import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.Tangle; +import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Pair; import java.io.*; import java.util.Collection; -import java.util.List; +import java.util.HashMap; import java.util.stream.Collectors; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * + * * Implementation of SpentAddressesProvider. * Addresses are saved/found on the {@link Tangle}. * The addresses will be written to a file called {@value #SNAPSHOT_SPENTADDRESSES_FILE}. @@ -33,7 +33,7 @@ public class SpentAddressesProviderImpl implements SpentAddressesProvider { private static final Logger log = LoggerFactory.getLogger(SpentAddressesProviderImpl.class); private static final String SNAPSHOT_SPENTADDRESSES_FILE = ".snapshot.spentaddresses"; - private Tangle tangle; + private RocksDBPersistenceProvider rocksDBPersistenceProvider; private SnapshotConfig config; @@ -43,22 +43,25 @@ public class SpentAddressesProviderImpl implements SpentAddressesProvider { * Creates a new instance of SpentAddressesProvider */ public SpentAddressesProviderImpl() { - + this.rocksDBPersistenceProvider = new RocksDBPersistenceProvider("spent-addresses-db", + "spent-addresses-log", 1000, + new HashMap>(1) + {{put("spent-addresses", SpentAddress.class);}}, null); } /** * Starts the SpentAddressesProvider by reading the previous spent addresses from file. * If {@value #SNAPSHOT_SPENTADDRESSES_FILE} already exists, these addresses will be read as well. - * - * @param tangle Tangle object which acts as a database interface + * * @param config The snapshot configuration used for file location * @return the current instance * @throws SpentAddressesException if we failed to create a file at the designated location */ - public SpentAddressesProviderImpl init(Tangle tangle, SnapshotConfig config) throws SpentAddressesException { - this.tangle = tangle; + public SpentAddressesProviderImpl init(SnapshotConfig config) + throws SpentAddressesException { this.config = config; this.localSnapshotAddressesFile = new File(config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE); + this.rocksDBPersistenceProvider.init(); readPreviousEpochsSpentAddresses(); if (localSnapshotAddressesFile.exists()) { @@ -114,7 +117,7 @@ private void readSpentAddressesFromStream(InputStream in) throws SpentAddressesE @Override public boolean containsAddress(Hash addressHash) throws SpentAddressesException { try { - return ((SpentAddress) tangle.load(SpentAddress.class, addressHash)).exists; + return ((SpentAddress) rocksDBPersistenceProvider.get(SpentAddress.class, addressHash)).exists; } catch (Exception e) { throw new SpentAddressesException(e); } @@ -123,19 +126,19 @@ public boolean containsAddress(Hash addressHash) throws SpentAddressesException @Override public void addAddress(Hash addressHash) throws SpentAddressesException { try { - tangle.save(new SpentAddress(), addressHash); + rocksDBPersistenceProvider.save(new SpentAddress(), addressHash); } catch (Exception e) { throw new SpentAddressesException(e); } } - + @Override public void addAddressesBatch(Collection addressHash) throws SpentAddressesException { try { // Its bytes are always new byte[0], therefore identical in storage SpentAddress spentAddressModel = new SpentAddress(); - - tangle.saveBatch(addressHash + + rocksDBPersistenceProvider.saveBatch(addressHash .stream() .map(address -> new Pair(address, spentAddressModel)) .collect(Collectors.toList()) @@ -156,6 +159,8 @@ public void writeSpentAddressesToDisk(String basePath) throws SpentAddressesExce } private List getAllSpentAddresses() { - return tangle.loadAllKeysFromTable(SpentAddress.class, HashFactory.ADDRESS::create); + return rocksDBPersistenceProvider.loadAllKeysFromTable(SpentAddress.class).stream() + .map(HashFactory.ADDRESS::create) + .collect(Collectors.toList()); } } From b8af7e94763a9ec97af31fdd931719aa9a790fb5 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Wed, 9 Jan 2019 13:28:21 +0200 Subject: [PATCH 37/63] Do not write spent addresses to files --- .../snapshot/impl/SnapshotServiceImpl.java | 4 +- .../SpentAddressesProvider.java | 13 +---- .../impl/SpentAddressesProviderImpl.java | 58 ++++--------------- .../impl/SpentAddressesServiceImpl.java | 4 +- .../rocksDB/RocksDBPersistenceProvider.java | 6 +- .../RocksDBPersistenceProviderTest.java | 2 +- 6 files changed, 20 insertions(+), 67 deletions(-) diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java index 40a865fa70..f34d24ca37 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java @@ -499,9 +499,7 @@ private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot ne try { spentAddressesService.persistSpentAddresses(snapshotProvider.getInitialSnapshot().getIndex(), newSnapshot.getIndex()); - - spentAddressesProvider.writeSpentAddressesToDisk(config.getLocalSnapshotsBasePath() + - ".snapshot.spentaddresses"); + } catch (Exception e) { throw new SnapshotException(e); } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java index e122701fdd..f37e9f6f47 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java @@ -24,7 +24,7 @@ public interface SpentAddressesProvider { * @param addressHash the address which we want to mark. * @throws SpentAddressesException If the provider fails to add the address */ - void addAddress(Hash addressHash) throws SpentAddressesException; + void saveAddress(Hash addressHash) throws SpentAddressesException; /** * Mark all addresses as spent. @@ -32,13 +32,6 @@ public interface SpentAddressesProvider { * @param addressHashes The addresses we want to mark * @throws SpentAddressesException If the provider fails to add an address */ - void addAddressesBatch(Collection addressHashes) throws SpentAddressesException; - - /** - * Writes all currently known spent addresses to disk. - * - * @param basePath the base path of the file we are going to write to. - * @throws SpentAddressesException If the provider fails to write the addresses to disk - */ - void writeSpentAddressesToDisk(String basePath) throws SpentAddressesException; + void saveAddressesBatch(Collection addressHashes) throws SpentAddressesException; + } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 6cd10c4972..9796cc2784 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -31,14 +31,11 @@ */ public class SpentAddressesProviderImpl implements SpentAddressesProvider { private static final Logger log = LoggerFactory.getLogger(SpentAddressesProviderImpl.class); - private static final String SNAPSHOT_SPENTADDRESSES_FILE = ".snapshot.spentaddresses"; private RocksDBPersistenceProvider rocksDBPersistenceProvider; private SnapshotConfig config; - private File localSnapshotAddressesFile; - /** * Creates a new instance of SpentAddressesProvider */ @@ -50,8 +47,7 @@ public SpentAddressesProviderImpl() { } /** - * Starts the SpentAddressesProvider by reading the previous spent addresses from file. - * If {@value #SNAPSHOT_SPENTADDRESSES_FILE} already exists, these addresses will be read as well. + * Starts the SpentAddressesProvider by reading the previous spent addresses from files. * * @param config The snapshot configuration used for file location * @return the current instance @@ -60,21 +56,13 @@ public SpentAddressesProviderImpl() { public SpentAddressesProviderImpl init(SnapshotConfig config) throws SpentAddressesException { this.config = config; - this.localSnapshotAddressesFile = new File(config.getLocalSnapshotsBasePath() + SNAPSHOT_SPENTADDRESSES_FILE); - this.rocksDBPersistenceProvider.init(); - - readPreviousEpochsSpentAddresses(); - if (localSnapshotAddressesFile.exists()) { - readLocalSpentAddresses(); + try { + this.rocksDBPersistenceProvider.init(); + readPreviousEpochsSpentAddresses(); } - else { - try { - localSnapshotAddressesFile.createNewFile(); - } catch (IOException e) { - throw new SpentAddressesException("Failed to create missing " + localSnapshotAddressesFile.getName(), e); - } + catch (Exception e) { + throw new SpentAddressesException("There is a problem with accessing stored spent addresses", e); } - return this; } @@ -88,26 +76,16 @@ private void readPreviousEpochsSpentAddresses() { readSpentAddressesFromStream( SpentAddressesProviderImpl.class.getResourceAsStream(previousEpochsSpentAddressesFile)); } catch (SpentAddressesException e) { - log.error("failed to read spent addresses from " + previousEpochsSpentAddressesFile, e); + log.error("Failed to read spent addresses from " + previousEpochsSpentAddressesFile, e); } } } - private void readLocalSpentAddresses() { - - try { - readSpentAddressesFromStream( - new FileInputStream(localSnapshotAddressesFile)); - } catch (Exception e) { - log.error("failed to read spent addresses from " + localSnapshotAddressesFile.getPath(), e); - } - } - private void readSpentAddressesFromStream(InputStream in) throws SpentAddressesException { try (BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { String line; while ((line = reader.readLine()) != null) { - addAddress(HashFactory.ADDRESS.create(line)); + saveAddress(HashFactory.ADDRESS.create(line)); } } catch (Exception e) { throw new SpentAddressesException(e); @@ -124,7 +102,7 @@ public boolean containsAddress(Hash addressHash) throws SpentAddressesException } @Override - public void addAddress(Hash addressHash) throws SpentAddressesException { + public void saveAddress(Hash addressHash) throws SpentAddressesException { try { rocksDBPersistenceProvider.save(new SpentAddress(), addressHash); } catch (Exception e) { @@ -133,7 +111,7 @@ public void addAddress(Hash addressHash) throws SpentAddressesException { } @Override - public void addAddressesBatch(Collection addressHash) throws SpentAddressesException { + public void saveAddressesBatch(Collection addressHash) throws SpentAddressesException { try { // Its bytes are always new byte[0], therefore identical in storage SpentAddress spentAddressModel = new SpentAddress(); @@ -147,20 +125,4 @@ public void addAddressesBatch(Collection addressHash) throws SpentAddresse throw new SpentAddressesException(e); } } - - @Override - public void writeSpentAddressesToDisk(String basePath) throws SpentAddressesException { - try { - Collection addressHashes = getAllSpentAddresses(); - FileUtils.writeLines(localSnapshotAddressesFile, addressHashes, false); - } catch (Exception e) { - throw new SpentAddressesException("Failed to dump spent addresses to disk", e); - } - } - - private List getAllSpentAddresses() { - return rocksDBPersistenceProvider.loadAllKeysFromTable(SpentAddress.class).stream() - .map(HashFactory.ADDRESS::create) - .collect(Collectors.toList()); - } } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java index 15605e2c79..a18a1fdead 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -103,7 +103,7 @@ public void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) //Can only throw runtime exceptions in streams try { - spentAddressesProvider.addAddressesBatch(addressesToCheck.stream() + spentAddressesProvider.saveAddressesBatch(addressesToCheck.stream() .filter(ThrowingPredicate.unchecked(this::wasAddressSpentFrom)) .collect(Collectors.toList())); } catch (RuntimeException e) { @@ -122,7 +122,7 @@ public void persistSpentAddresses(Collection transactions) .filter(ThrowingPredicate.unchecked(this::wasTransactionSpentFrom)) .map(TransactionViewModel::getAddressHash).collect(Collectors.toSet()); - spentAddressesProvider.addAddressesBatch(spentAddresses); + spentAddressesProvider.saveAddressesBatch(spentAddresses); } catch (RuntimeException e) { throw new SpentAddressesException("Exception while persisting spent addresses", e); } diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java index 93c91e7239..7f6a46e9d3 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java @@ -58,7 +58,7 @@ public RocksDBPersistenceProvider(String dbPath, String logPath, int cacheSize, } @Override - public void init() { + public void init() throws Exception { log.info("Initializing Database Backend... "); initDB(dbPath, logPath, columnFamilies); available = true; @@ -394,7 +394,7 @@ public void restoreBackup(String path, String logPath) throws Exception { initDB(path, logPath, columnFamilies); } - private void initDB(String path, String logPath, Map> columnFamilies) { + private void initDB(String path, String logPath, Map> columnFamilies) throws Exception { try { try { RocksDB.loadLibrary(); @@ -472,8 +472,8 @@ private void initDB(String path, String logPath, Map Date: Wed, 9 Jan 2019 17:30:43 +0200 Subject: [PATCH 38/63] fix log message to display the db path --- .../iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java index 7f6a46e9d3..731034bb7c 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java @@ -59,7 +59,7 @@ public RocksDBPersistenceProvider(String dbPath, String logPath, int cacheSize, @Override public void init() throws Exception { - log.info("Initializing Database Backend... "); + log.info("Initializing Database on " + dbPath); initDB(dbPath, logPath, columnFamilies); available = true; log.info("RocksDB persistence provider initialized."); From f2bad0937064096290f0b091bea22e9964691ec3 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Wed, 9 Jan 2019 18:42:11 +0100 Subject: [PATCH 39/63] added lastSnapshottedMilestoneIndex --- src/main/java/com/iota/iri/service/API.java | 9 ++++++++- .../iri/service/dto/GetNodeInfoResponse.java | 18 +++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 932310f0c8..a20bbed809 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -874,17 +874,24 @@ private AbstractResponse interruptAttachingToTangleStatement(){ **/ private AbstractResponse getNodeInfoStatement(){ String name = instance.configuration.isTestnet() ? IRI.TESTNET_NAME : IRI.MAINNET_NAME; - return GetNodeInfoResponse.create(name, IRI.VERSION, + return GetNodeInfoResponse.create( + name, + IRI.VERSION, Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().freeMemory(), System.getProperty("java.version"), + Runtime.getRuntime().maxMemory(), Runtime.getRuntime().totalMemory(), instance.latestMilestoneTracker.getLatestMilestoneHash(), instance.latestMilestoneTracker.getLatestMilestoneIndex(), + instance.snapshotProvider.getLatestSnapshot().getHash(), instance.snapshotProvider.getLatestSnapshot().getIndex(), + instance.snapshotProvider.getInitialSnapshot().getIndex(), + instance.snapshotProvider.getLatestSnapshot().getInitialIndex(), + instance.node.howManyNeighbors(), instance.node.queuedTransactionsSize(), System.currentTimeMillis(), diff --git a/src/main/java/com/iota/iri/service/dto/GetNodeInfoResponse.java b/src/main/java/com/iota/iri/service/dto/GetNodeInfoResponse.java index f60e662671..0dd95772b0 100644 --- a/src/main/java/com/iota/iri/service/dto/GetNodeInfoResponse.java +++ b/src/main/java/com/iota/iri/service/dto/GetNodeInfoResponse.java @@ -75,6 +75,12 @@ public class GetNodeInfoResponse extends AbstractResponse { */ private int milestoneStartIndex; + /** + * The index of the milestone used in the latest snapshot. + * This is the most recent milestone in the entire snapshot + */ + private int lastSnapshottedMilestoneIndex; + /** * Number of neighbors this node is directly connected with. */ @@ -132,6 +138,7 @@ public class GetNodeInfoResponse extends AbstractResponse { * @param latestSolidSubtangleMilestone {@link #latestSolidSubtangleMilestone} * @param latestSolidSubtangleMilestoneIndex {@link #latestSolidSubtangleMilestoneIndex} * @param milestoneStartIndex {@link #milestoneStartIndex} + * @param lastSnapshottedMilestoneIndex {@link #lastSnapshottedMilestoneIndex} * @param neighbors {@link #neighbors} * @param packetsQueueSize {@link #packetsQueueSize} * @param currentTimeMillis {@link #time} @@ -144,7 +151,7 @@ public class GetNodeInfoResponse extends AbstractResponse { public static AbstractResponse create(String appName, String appVersion, int jreAvailableProcessors, long jreFreeMemory, String jreVersion, long maxMemory, long totalMemory, Hash latestMilestone, int latestMilestoneIndex, Hash latestSolidSubtangleMilestone, int latestSolidSubtangleMilestoneIndex, int milestoneStartIndex, - int neighbors, int packetsQueueSize, long currentTimeMillis, int tips, + int lastSnapshottedMilestoneIndex, int neighbors, int packetsQueueSize, long currentTimeMillis, int tips, int numberOfTransactionsToRequest, String[] features, String coordinatorAddress) { final GetNodeInfoResponse res = new GetNodeInfoResponse(); res.appName = appName; @@ -162,6 +169,7 @@ public static AbstractResponse create(String appName, String appVersion, int jre res.latestSolidSubtangleMilestoneIndex = latestSolidSubtangleMilestoneIndex; res.milestoneStartIndex = milestoneStartIndex; + res.lastSnapshottedMilestoneIndex = lastSnapshottedMilestoneIndex; res.neighbors = neighbors; res.packetsQueueSize = packetsQueueSize; @@ -269,6 +277,14 @@ public int getLatestSolidSubtangleMilestoneIndex() { public int getMilestoneStartIndex() { return milestoneStartIndex; } + + /** + * + * @return {@link #lastSnapshottedMilestoneIndex} + */ + public int getLastSnapshottedMilestoneIndex() { + return lastSnapshottedMilestoneIndex; + } /** * From 91aa64a24cb1f2dfe67cbf0bd8884f0119d1201a Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 01:36:30 +0200 Subject: [PATCH 40/63] Move column families to Tangle --- src/main/java/com/iota/iri/Iota.java | 21 +++------------- .../java/com/iota/iri/storage/Tangle.java | 24 +++++++++++++++---- .../rocksDB/RocksDBPersistenceProvider.java | 10 ++++---- .../com/iota/iri/BundleValidatorTest.java | 4 ++-- .../iota/iri/TransactionValidatorTest.java | 2 +- .../dbbenchmark/states/DbState.java | 2 +- .../iri/controllers/BundleViewModelTest.java | 2 +- .../controllers/MilestoneViewModelTest.java | 2 +- .../controllers/TransactionViewModelTest.java | 2 +- .../impl/CumulativeWeightCalculatorTest.java | 2 +- .../tipselection/impl/RatingOneTest.java | 2 +- .../tipselection/impl/TailFinderImplTest.java | 2 +- .../impl/WalkValidatorImplTest.java | 2 +- .../tipselection/impl/WalkerAlphaTest.java | 2 +- .../java/com/iota/iri/storage/TangleTest.java | 2 +- .../RocksDBPersistenceProviderTest.java | 3 ++- 16 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 59ec4b1c9f..ba7878b63d 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -25,10 +25,7 @@ import com.iota.iri.service.tipselection.impl.*; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner; -import com.iota.iri.storage.Indexable; -import com.iota.iri.storage.Persistable; -import com.iota.iri.storage.Tangle; -import com.iota.iri.storage.ZmqPublishProvider; +import com.iota.iri.storage.*; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import com.iota.iri.utils.Pair; import com.iota.iri.zmq.MessageQ; @@ -75,18 +72,6 @@ */ public class Iota { private static final Logger log = LoggerFactory.getLogger(Iota.class); - public static final HashMap> COLUMN_FAMILIES = - new LinkedHashMap>() {{ - put("transaction", Transaction.class); - put("milestone", Milestone.class); - put("stateDiff", StateDiff.class); - put("address", Address.class); - put("approvee", Approvee.class); - put("bundle", Bundle.class); - put("obsoleteTag", ObsoleteTag.class); - put("tag", Tag.class); - }}; - public static final Pair> METADATA_COLUMN_FAMILY = new Pair<>("transaction-metadata", Transaction.class); public final SpentAddressesProviderImpl spentAddressesProvider; @@ -300,8 +285,8 @@ private void initializeTangle() { configuration.getDbPath(), configuration.getDbLogPath(), configuration.getDbCacheSize(), - COLUMN_FAMILIES, - METADATA_COLUMN_FAMILY) + Tangle.COLUMN_FAMILIES, + Tangle.METADATA_COLUMN_FAMILY) ); break; } diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index 73b994d9d3..cd57bbae2c 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -1,14 +1,14 @@ package com.iota.iri.storage; +import com.iota.iri.model.StateDiff; +import com.iota.iri.model.persistables.*; import com.iota.iri.utils.Pair; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,8 +18,24 @@ public class Tangle { private static final Logger log = LoggerFactory.getLogger(Tangle.class); + public static final Map> COLUMN_FAMILIES = + new LinkedHashMap>() {{ + put("transaction", Transaction.class); + put("milestone", Milestone.class); + put("stateDiff", StateDiff.class); + put("address", Address.class); + put("approvee", Approvee.class); + put("bundle", Bundle.class); + put("obsoleteTag", ObsoleteTag.class); + put("tag", Tag.class); + }}; + + public static final Map.Entry> METADATA_COLUMN_FAMILY = + new AbstractMap.SimpleImmutableEntry<>("transaction-metadata", Transaction.class); + private final List persistenceProviders = new ArrayList<>(); + public void addPersistenceProvider(PersistenceProvider provider) { this.persistenceProviders.add(provider); } diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java index 731034bb7c..a7646c2dc1 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java @@ -35,7 +35,7 @@ public class RocksDBPersistenceProvider implements PersistenceProvider { private final String logPath; private final int cacheSize; private final Map> columnFamilies; - private final Pair> metadataColumnFamily; + private final Map.Entry> metadataColumnFamily; private Map, ColumnFamilyHandle> classTreeMap; private Map, ColumnFamilyHandle> metadataReference = Collections.emptyMap(); @@ -48,7 +48,7 @@ public class RocksDBPersistenceProvider implements PersistenceProvider { public RocksDBPersistenceProvider(String dbPath, String logPath, int cacheSize, Map> columnFamilies, - Pair> metadataColumnFamily) { + Map.Entry> metadataColumnFamily) { this.dbPath = dbPath; this.logPath = logPath; this.cacheSize = cacheSize; @@ -461,7 +461,7 @@ private void initDB(String path, String logPath, Map(); } @@ -479,7 +479,7 @@ private void initDB(String path, String logPath, Map columnFamilyDescriptors) throws Exception { Map, ColumnFamilyHandle> classMap = new LinkedHashMap<>(); - String mcfName = metadataColumnFamily == null ? "" : metadataColumnFamily.low; + String mcfName = metadataColumnFamily == null ? "" : metadataColumnFamily.getKey(); //skip default column int i = 1; for (; i < columnFamilyDescriptors.size(); i++) { @@ -487,7 +487,7 @@ private void initClassTreeMap(List columnFamilyDescripto String name = new String(columnFamilyDescriptors.get(i).columnFamilyName()); if (name.equals(mcfName)) { Map, ColumnFamilyHandle> metadataRef = new HashMap<>(); - metadataRef.put(metadataColumnFamily.hi, columnFamilyHandles.get(i)); + metadataRef.put(metadataColumnFamily.getValue(), columnFamilyHandles.get(i)); metadataReference = MapUtils.unmodifiableMap(metadataRef); } else { diff --git a/src/test/java/com/iota/iri/BundleValidatorTest.java b/src/test/java/com/iota/iri/BundleValidatorTest.java index a3a09444a4..c6eadc4d82 100644 --- a/src/test/java/com/iota/iri/BundleValidatorTest.java +++ b/src/test/java/com/iota/iri/BundleValidatorTest.java @@ -30,8 +30,8 @@ public static void setUp() throws Exception { logFolder.create(); tangle.addPersistenceProvider( new RocksDBPersistenceProvider(dbFolder.getRoot().getAbsolutePath(), - logFolder.getRoot().getAbsolutePath(), 1000, Iota.COLUMN_FAMILIES, - Iota.METADATA_COLUMN_FAMILY)); + logFolder.getRoot().getAbsolutePath(), 1000, Tangle.COLUMN_FAMILIES, + Tangle.METADATA_COLUMN_FAMILY)); tangle.init(); snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); } diff --git a/src/test/java/com/iota/iri/TransactionValidatorTest.java b/src/test/java/com/iota/iri/TransactionValidatorTest.java index 44b6734568..fcf9929464 100644 --- a/src/test/java/com/iota/iri/TransactionValidatorTest.java +++ b/src/test/java/com/iota/iri/TransactionValidatorTest.java @@ -39,7 +39,7 @@ public static void setUp() throws Exception { snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); tangle.addPersistenceProvider( new RocksDBPersistenceProvider( - dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); tangle.init(); TipsViewModel tipsViewModel = new TipsViewModel(); MessageQ messageQ = Mockito.mock(MessageQ.class); diff --git a/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java b/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java index 26e6231417..130f8c9535 100644 --- a/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java +++ b/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java @@ -43,7 +43,7 @@ public void setup() throws Exception { } logFolder.mkdirs(); PersistenceProvider dbProvider = new RocksDBPersistenceProvider( - dbFolder.getAbsolutePath(), logFolder.getAbsolutePath(), BaseIotaConfig.Defaults.DB_CACHE_SIZE, Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); + dbFolder.getAbsolutePath(), logFolder.getAbsolutePath(), BaseIotaConfig.Defaults.DB_CACHE_SIZE, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); dbProvider.init(); tangle = new Tangle(); snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); diff --git a/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java b/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java index b838a09542..ed3a7d9140 100644 --- a/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java @@ -23,7 +23,7 @@ public void setUp() throws Exception { RocksDBPersistenceProvider rocksDBPersistenceProvider; rocksDBPersistenceProvider = new RocksDBPersistenceProvider( dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, - Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); + Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); tangle.addPersistenceProvider(rocksDBPersistenceProvider); tangle.init(); diff --git a/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java b/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java index e602c77553..ad2ab0b300 100644 --- a/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java @@ -27,7 +27,7 @@ public void setUpTest() throws Exception { RocksDBPersistenceProvider rocksDBPersistenceProvider; rocksDBPersistenceProvider = new RocksDBPersistenceProvider( dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, - Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); + Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); tangle.addPersistenceProvider(rocksDBPersistenceProvider); tangle.init(); } diff --git a/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java b/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java index fddc1863f8..43912a8db1 100644 --- a/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java @@ -44,7 +44,7 @@ public static void setUp() throws Exception { RocksDBPersistenceProvider rocksDBPersistenceProvider; rocksDBPersistenceProvider = new RocksDBPersistenceProvider( dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, - Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); + Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); tangle.addPersistenceProvider(rocksDBPersistenceProvider); tangle.init(); snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java index ade3ea34ce..1754a6fe4d 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java @@ -50,7 +50,7 @@ public static void setUp() throws Exception { dbFolder.create(); logFolder.create(); tangle.addPersistenceProvider( new RocksDBPersistenceProvider( - dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); + dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); tangle.init(); cumulativeWeightCalculator = new CumulativeWeightCalculator(tangle, snapshotProvider); } diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java index 45777f3322..f87271061b 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java @@ -42,7 +42,7 @@ public static void setUp() throws Exception { logFolder.create(); tangle.addPersistenceProvider( new RocksDBPersistenceProvider( dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, - Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); + Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); tangle.init(); rating = new RatingOne(tangle); } diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java index ad070fe292..df1467f340 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java @@ -46,7 +46,7 @@ public static void setUp() throws Exception { logFolder.create(); tangle.addPersistenceProvider( new RocksDBPersistenceProvider( dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, - Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); + Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); tangle.init(); } diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java index 14a6f7ffaf..91d474d743 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java @@ -52,7 +52,7 @@ public static void setUp() throws Exception { logFolder.create(); tangle.addPersistenceProvider( new RocksDBPersistenceProvider( dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, - Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); + Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); tangle.init(); } diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java index 8741f6e59e..28fb1a08d5 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java @@ -56,7 +56,7 @@ public static void setUp() throws Exception { logFolder.create(); tangle.addPersistenceProvider( new RocksDBPersistenceProvider( dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, - Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY)); + Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY)); tangle.init(); MessageQ messageQ = Mockito.mock(MessageQ.class); diff --git a/src/test/java/com/iota/iri/storage/TangleTest.java b/src/test/java/com/iota/iri/storage/TangleTest.java index d10f57225b..385db4db73 100644 --- a/src/test/java/com/iota/iri/storage/TangleTest.java +++ b/src/test/java/com/iota/iri/storage/TangleTest.java @@ -34,7 +34,7 @@ public void setUp() throws Exception { RocksDBPersistenceProvider rocksDBPersistenceProvider; rocksDBPersistenceProvider = new RocksDBPersistenceProvider( dbFolder.getRoot().getAbsolutePath(), logFolder.getRoot().getAbsolutePath(),1000, - Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); + Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); tangle.addPersistenceProvider(rocksDBPersistenceProvider); tangle.init(); snapshotProvider = new SnapshotProviderImpl().init(new MainnetConfig()); diff --git a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java index dfe8a65010..cc8a8c9205 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java @@ -5,6 +5,7 @@ import com.iota.iri.model.persistables.Transaction; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.Tangle; import com.iota.iri.utils.Pair; import org.apache.commons.io.FileUtils; import org.junit.*; @@ -25,7 +26,7 @@ public class RocksDBPersistenceProviderTest { @BeforeClass public static void setUpDb() throws Exception { rocksDBPersistenceProvider = new RocksDBPersistenceProvider( - dbPath, dbLogPath,1000, Iota.COLUMN_FAMILIES, Iota.METADATA_COLUMN_FAMILY); + dbPath, dbLogPath,1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); rocksDBPersistenceProvider.init(); } From bd60a4a9988be8a28f18437bc5eece757d5ff8b6 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 01:46:55 +0200 Subject: [PATCH 41/63] delete javadoc --- .../service/spentaddresses/impl/SpentAddressesProviderImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 9796cc2784..77a46a6e90 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -25,7 +25,6 @@ * * Implementation of SpentAddressesProvider. * Addresses are saved/found on the {@link Tangle}. - * The addresses will be written to a file called {@value #SNAPSHOT_SPENTADDRESSES_FILE}. * The folder location is provided by {@link IotaConfig#getLocalSnapshotsBasePath()} * */ From 35fe8d31c966ec8e64515122420b1f2e7dc50319 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 01:53:14 +0200 Subject: [PATCH 42/63] throw exception --- .../spentaddresses/impl/SpentAddressesProviderImpl.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 77a46a6e90..c5a84587f4 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -65,18 +65,14 @@ public SpentAddressesProviderImpl init(SnapshotConfig config) return this; } - private void readPreviousEpochsSpentAddresses() { + private void readPreviousEpochsSpentAddresses() throws SpentAddressesException { if (config.isTestnet()) { return; } for (String previousEpochsSpentAddressesFile : config.getPreviousEpochSpentAddressesFiles().split(" ")) { - try { readSpentAddressesFromStream( SpentAddressesProviderImpl.class.getResourceAsStream(previousEpochsSpentAddressesFile)); - } catch (SpentAddressesException e) { - log.error("Failed to read spent addresses from " + previousEpochsSpentAddressesFile, e); - } } } @@ -87,7 +83,7 @@ private void readSpentAddressesFromStream(InputStream in) throws SpentAddressesE saveAddress(HashFactory.ADDRESS.create(line)); } } catch (Exception e) { - throw new SpentAddressesException(e); + throw new SpentAddressesException("Failed to read or save spent address", e); } } From 8eaf886aa9bbb511e09d05978100561e86b61424 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 01:27:26 +0100 Subject: [PATCH 43/63] Added queue getter in TransactionPruner, allowing us to get youngestFullyPruned --- src/main/java/com/iota/iri/service/API.java | 11 ++++++++++- .../transactionpruning/TransactionPruner.java | 10 ++++++++++ .../async/AsyncTransactionPruner.java | 13 +++++++++++++ .../async/MilestonePrunerJobQueue.java | 14 ++++++++++++-- .../jobs/MilestonePrunerJob.java | 3 +++ 5 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index a20bbed809..b72844cfd4 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -18,6 +18,8 @@ import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.tipselection.TipSelector; import com.iota.iri.service.tipselection.impl.WalkValidatorImpl; +import com.iota.iri.service.transactionpruning.async.MilestonePrunerJobQueue; +import com.iota.iri.service.transactionpruning.jobs.MilestonePrunerJob; import com.iota.iri.utils.Converter; import com.iota.iri.utils.IotaIOUtils; import com.iota.iri.utils.MapIdentityManager; @@ -874,6 +876,13 @@ private AbstractResponse interruptAttachingToTangleStatement(){ **/ private AbstractResponse getNodeInfoStatement(){ String name = instance.configuration.isTestnet() ? IRI.TESTNET_NAME : IRI.MAINNET_NAME; + + // Note: When the MilestonePrunerQueue is processing a job, it is possible that this index is out of date. + // Chances of this increase when a large localSnapshotsIntervalSynced is chosen. + int initialIndex = instance.transactionPruner + .getJobQueueByQueueClass(MilestonePrunerJobQueue.class) + .getYoungestFullyCleanedMilestoneIndex(); + return GetNodeInfoResponse.create( name, IRI.VERSION, @@ -889,7 +898,7 @@ private AbstractResponse getNodeInfoStatement(){ instance.snapshotProvider.getLatestSnapshot().getHash(), instance.snapshotProvider.getLatestSnapshot().getIndex(), - instance.snapshotProvider.getInitialSnapshot().getIndex(), + initialIndex, instance.snapshotProvider.getLatestSnapshot().getInitialIndex(), instance.node.howManyNeighbors(), diff --git a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java index ed6c62de3a..0323a138f9 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java @@ -1,5 +1,7 @@ package com.iota.iri.service.transactionpruning; +import com.iota.iri.service.transactionpruning.async.JobQueue; + /** * Represents the manager for the cleanup jobs that are issued by the * {@link com.iota.iri.service.snapshot.LocalSnapshotManager} in connection with local snapshots and eventually other @@ -19,6 +21,14 @@ public interface TransactionPruner { * @throws TransactionPruningException if anything goes wrong while adding the job */ void addJob(TransactionPrunerJob job) throws TransactionPruningException; + + /** + * Finds the specific JobQueue instance from the current queues. + * + * @param jobQueueType the class which extends of the JobQueue + * @return The JobQueue, or null if it does not exist + */ + T getJobQueueByQueueClass(Class jobQueueType); /** * This method executes all jobs that where added to the {@link TransactionPruner} through diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index dd7784b844..3cfd578185 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -18,6 +18,7 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; @@ -165,6 +166,18 @@ public void addJob(TransactionPrunerJob job) throws TransactionPruningException saveState(); } + + /** + * {@inheritDoc} + * + * Gets the current job from its corresponding queue. + */ + @Override + public T getJobQueueByQueueClass(Class jobQueueType) { + return (T) jobQueues.values().stream(). + filter(jobQueue -> jobQueue.getClass().equals(jobQueueType)). + findFirst().orElse(null); + } /** * {@inheritDoc} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java b/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java index 81b8ca0e21..e3bcdabecd 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java @@ -11,13 +11,13 @@ import java.util.stream.Stream; /** - * Represents a queue of {@link MilestonePrunerJob}s thar are being executed by the {@link AsyncTransactionPruner}. + * Represents a queue of {@link MilestonePrunerJob}s that are being executed by the {@link AsyncTransactionPruner}. * * The {@link AsyncTransactionPruner} uses a separate queue for every job type, to be able to adjust the processing * logic based on the type of the job. */ public class MilestonePrunerJobQueue implements JobQueue { - /** + /** f * Holds the youngest (highest) milestone index that was successfully cleaned (gets updated when a job finishes). */ private int youngestFullyCleanedMilestoneIndex; @@ -164,4 +164,14 @@ private boolean adjustedRangeCoversNewMilestone(MilestonePrunerJob job, Mileston return true; } + + /** + * Gets the youngest cleaned milestone index. + * This is not guaranteed to be the current youngest in the database, since the current job could be making changes. + * + * @return The milestone index of the current youngest fully cleaned milestone in the provider + */ + public int getYoungestFullyCleanedMilestoneIndex() { + return youngestFullyCleanedMilestoneIndex; + } } diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index 8974c6a981..838f418cf8 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -69,6 +69,7 @@ public static MilestonePrunerJob parse(String input) throws TransactionPruningEx * get restored from a state file, because we start cleaning up at the {@link #startingIndex}. * * @param startingIndex milestone index that defines where to start cleaning up + * @param targetIndex milestone index which defines the end of this pruning job */ public MilestonePrunerJob(int startingIndex, int targetIndex) { this(startingIndex, startingIndex, targetIndex); @@ -88,6 +89,7 @@ public MilestonePrunerJob(int startingIndex, int targetIndex) { * * @param startingIndex milestone index that defines where to start cleaning up * @param currentIndex milestone index that defines the next milestone that should be cleaned up by this job + * @param targetIndex milestone index which defines the end of this pruning job */ private MilestonePrunerJob(int startingIndex, int currentIndex, int targetIndex) { setStartingIndex(startingIndex); @@ -125,6 +127,7 @@ public void process() throws TransactionPruningException { } getTransactionPruner().saveState(); + getSnapshot().setIndex(getCurrentIndex()); } } catch (TransactionPruningException e) { setStatus(TransactionPrunerJobStatus.FAILED); From a94ba97546959b83f6bc7db27cb56a0b33dd232a Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 01:27:45 +0100 Subject: [PATCH 44/63] Added lastSnapshottedMilestoneIndex to test --- src/test/java/com/iota/iri/service/APIIntegrationTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/iota/iri/service/APIIntegrationTests.java b/src/test/java/com/iota/iri/service/APIIntegrationTests.java index 23394ac05f..7f205c8245 100644 --- a/src/test/java/com/iota/iri/service/APIIntegrationTests.java +++ b/src/test/java/com/iota/iri/service/APIIntegrationTests.java @@ -173,6 +173,7 @@ public void shouldTestGetNodeInfo() { body(containsString("latestSolidSubtangleMilestone")). body(containsString("latestSolidSubtangleMilestoneIndex")). body(containsString("milestoneStartIndex")). + body(containsString("lastSnapshottedMilestoneIndex")). body(containsString("neighbors")). body(containsString("packetsQueueSize")). body(containsString("time")). From 943ab94068963073ae1aed135096dc129743f900 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 01:46:37 +0100 Subject: [PATCH 45/63] Added check for MilestonePruner, and fallback value --- src/main/java/com/iota/iri/service/API.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index b72844cfd4..2abf570131 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -879,9 +879,11 @@ private AbstractResponse getNodeInfoStatement(){ // Note: When the MilestonePrunerQueue is processing a job, it is possible that this index is out of date. // Chances of this increase when a large localSnapshotsIntervalSynced is chosen. - int initialIndex = instance.transactionPruner - .getJobQueueByQueueClass(MilestonePrunerJobQueue.class) - .getYoungestFullyCleanedMilestoneIndex(); + // If pruning is off, return -1 + MilestonePrunerJobQueue job = instance.transactionPruner.getJobQueueByQueueClass(MilestonePrunerJobQueue.class); + int initialIndex = job != null ? + job.getYoungestFullyCleanedMilestoneIndex() : + instance.snapshotProvider.getInitialSnapshot().getInitialIndex(); return GetNodeInfoResponse.create( name, From 6b971d281d12d5edaa11cdbc3e4e3bbb2bf4752f Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 10:37:24 +0200 Subject: [PATCH 46/63] remove imports --- src/main/java/com/iota/iri/Iota.java | 4 ---- src/main/java/com/iota/iri/storage/Tangle.java | 1 - .../com/iota/iri/benchmarks/dbbenchmark/states/DbState.java | 1 - .../java/com/iota/iri/controllers/BundleViewModelTest.java | 1 - .../java/com/iota/iri/controllers/MilestoneViewModelTest.java | 1 - .../com/iota/iri/controllers/TransactionViewModelTest.java | 1 - .../tipselection/impl/CumulativeWeightCalculatorTest.java | 1 - .../com/iota/iri/service/tipselection/impl/RatingOneTest.java | 1 - .../iri/service/tipselection/impl/TailFinderImplTest.java | 1 - .../iri/service/tipselection/impl/WalkValidatorImplTest.java | 1 - .../iota/iri/service/tipselection/impl/WalkerAlphaTest.java | 1 - src/test/java/com/iota/iri/storage/TangleTest.java | 3 +-- .../iri/storage/rocksDB/RocksDBPersistenceProviderTest.java | 1 - 13 files changed, 1 insertion(+), 17 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index ba7878b63d..63fb820f5c 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -4,8 +4,6 @@ import com.iota.iri.conf.TipSelConfig; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.StateDiff; -import com.iota.iri.model.persistables.*; import com.iota.iri.network.Node; import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.UDPReceiver; @@ -31,8 +29,6 @@ import com.iota.iri.zmq.MessageQ; import java.security.SecureRandom; -import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import org.apache.commons.lang3.NotImplementedException; diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index cd57bbae2c..5e444bb173 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -8,7 +8,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java b/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java index 130f8c9535..fbadefae78 100644 --- a/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java +++ b/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java @@ -1,6 +1,5 @@ package com.iota.iri.benchmarks.dbbenchmark.states; -import com.iota.iri.Iota; import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.BaseIotaConfig; import com.iota.iri.conf.MainnetConfig; diff --git a/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java b/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java index ed3a7d9140..9bef020e95 100644 --- a/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/BundleViewModelTest.java @@ -1,6 +1,5 @@ package com.iota.iri.controllers; -import com.iota.iri.Iota; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import org.junit.After; diff --git a/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java b/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java index ad2ab0b300..452a4f219a 100644 --- a/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/MilestoneViewModelTest.java @@ -1,6 +1,5 @@ package com.iota.iri.controllers; -import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; diff --git a/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java b/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java index 43912a8db1..c84bc32cd1 100644 --- a/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java +++ b/src/test/java/com/iota/iri/controllers/TransactionViewModelTest.java @@ -1,6 +1,5 @@ package com.iota.iri.controllers; -import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.crypto.SpongeFactory; import com.iota.iri.model.Hash; diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java index 1754a6fe4d..df4360ef66 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/CumulativeWeightCalculatorTest.java @@ -1,7 +1,6 @@ package com.iota.iri.service.tipselection.impl; -import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.ApproveeViewModel; import com.iota.iri.controllers.TransactionViewModel; diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java index f87271061b..9617a84e57 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/RatingOneTest.java @@ -1,6 +1,5 @@ package com.iota.iri.service.tipselection.impl; -import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.HashId; diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java index df1467f340..52dc8b24f9 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/TailFinderImplTest.java @@ -1,6 +1,5 @@ package com.iota.iri.service.tipselection.impl; -import com.iota.iri.Iota; import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.TransactionViewModel; diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java index 91d474d743..d4025acfb8 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/WalkValidatorImplTest.java @@ -1,6 +1,5 @@ package com.iota.iri.service.tipselection.impl; -import com.iota.iri.Iota; import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.conf.TipSelConfig; diff --git a/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java b/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java index 28fb1a08d5..0ebbc6184c 100644 --- a/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java +++ b/src/test/java/com/iota/iri/service/tipselection/impl/WalkerAlphaTest.java @@ -1,6 +1,5 @@ package com.iota.iri.service.tipselection.impl; -import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; diff --git a/src/test/java/com/iota/iri/storage/TangleTest.java b/src/test/java/com/iota/iri/storage/TangleTest.java index 385db4db73..0d2b39e7d3 100644 --- a/src/test/java/com/iota/iri/storage/TangleTest.java +++ b/src/test/java/com/iota/iri/storage/TangleTest.java @@ -1,13 +1,12 @@ package com.iota.iri.storage; -import com.iota.iri.Iota; import com.iota.iri.conf.MainnetConfig; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.crypto.SpongeFactory; import com.iota.iri.model.TransactionHash; import com.iota.iri.model.persistables.Tag; import com.iota.iri.service.snapshot.SnapshotProvider; -import com.iota.iri.service.snapshot.impl.SnapshotProviderImpl; +import com.iota.iri.service .snapshot.impl.SnapshotProviderImpl; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; import org.junit.After; import org.junit.Assert; diff --git a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java index cc8a8c9205..f9f5e7b0ff 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java @@ -1,6 +1,5 @@ package com.iota.iri.storage.rocksDB; -import com.iota.iri.Iota; import com.iota.iri.model.IntegerIndex; import com.iota.iri.model.persistables.Transaction; import com.iota.iri.storage.Indexable; From 10241715693b395b51d1d6338d7d7735d0dc28a8 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 11:33:30 +0200 Subject: [PATCH 47/63] bump version to 1.6.0-RC15 --- DOCKER.md | 6 +++--- pom.xml | 2 +- src/main/java/com/iota/iri/IRI.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index 65c66b5af8..8aba8d7d6a 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -2,13 +2,13 @@ Run the official iotaledger/iri container, passing the mandatory -p option: -```docker run iotaledger/iri:v1.6.0-RC14 -p 14265``` +```docker run iotaledger/iri:v1.6.0-RC15 -p 14265``` This will get your a running IRI with its API listening on port 14265, no neighbours and an empty database. The IRI Docker container by default expects data at /iri/data. Use the `-v` option of the `docker run` command to mount volumes so to have persistent data. You can also pass more command line options to the docker run command and those will be passed to IRI. If you want to use a iri.ini file with the docker container, supposing it's stored under /path/to/conf/iri.ini on your docker host, then pass `-v /path/to/conf:/iri/conf` and add -c /iri/conf/iri.ini as docker run arguments. So for example the `docker run` command above would become: -```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC14 -p 14265 -c /iri/conf/iri.ini``` +```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC15 -p 14265 -c /iri/conf/iri.ini``` Please refer to the IRI documentation for further command line options and iri.ini options. @@ -61,7 +61,7 @@ ExecStart=/usr/bin/docker run \ -p 14265:14265 \ -p 15600:15600 \ -p 14600:14600/udp \ -iotaledger/iri:v1.6.0-RC14 \ +iotaledger/iri:v1.6.0-RC15 \ -p 14265 \ --zmq-enabled \ --testnet diff --git a/pom.xml b/pom.xml index 9efe4c834f..a6316fb2de 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.iota iri - 1.6.0-RC14 + 1.6.0-RC15 IRI IOTA Reference Implementation diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index 27b0de238c..ca3ac017af 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -42,7 +42,7 @@ public class IRI { public static final String MAINNET_NAME = "IRI"; public static final String TESTNET_NAME = "IRI Testnet"; - public static final String VERSION = "1.6.0-RC14"; + public static final String VERSION = "1.6.0-RC15"; /** * The entry point of IRI. From 747e94e6b17e21453ffa605269a81f5bb1f7fa90 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 12:58:59 +0100 Subject: [PATCH 48/63] Removed things from feedback/codacy --- src/main/java/com/iota/iri/service/API.java | 10 ++++------ .../async/AsyncTransactionPruner.java | 1 - .../async/MilestonePrunerJobQueue.java | 3 ++- .../transactionpruning/jobs/MilestonePrunerJob.java | 1 - 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 2abf570131..193470ea28 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -15,11 +15,9 @@ import com.iota.iri.model.persistables.Transaction; import com.iota.iri.network.Neighbor; import com.iota.iri.service.dto.*; -import com.iota.iri.service.snapshot.Snapshot; import com.iota.iri.service.tipselection.TipSelector; import com.iota.iri.service.tipselection.impl.WalkValidatorImpl; import com.iota.iri.service.transactionpruning.async.MilestonePrunerJobQueue; -import com.iota.iri.service.transactionpruning.jobs.MilestonePrunerJob; import com.iota.iri.utils.Converter; import com.iota.iri.utils.IotaIOUtils; import com.iota.iri.utils.MapIdentityManager; @@ -41,7 +39,6 @@ import java.nio.charset.StandardCharsets; import java.security.InvalidAlgorithmParameterException; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -880,9 +877,10 @@ private AbstractResponse getNodeInfoStatement(){ // Note: When the MilestonePrunerQueue is processing a job, it is possible that this index is out of date. // Chances of this increase when a large localSnapshotsIntervalSynced is chosen. // If pruning is off, return -1 - MilestonePrunerJobQueue job = instance.transactionPruner.getJobQueueByQueueClass(MilestonePrunerJobQueue.class); - int initialIndex = job != null ? - job.getYoungestFullyCleanedMilestoneIndex() : + MilestonePrunerJobQueue milestoneJobQueue = instance.transactionPruner + .getJobQueueByQueueClass(MilestonePrunerJobQueue.class); + int initialIndex = milestoneJobQueue != null ? + milestoneJobQueue.getYoungestFullyCleanedMilestoneIndex() : instance.snapshotProvider.getInitialSnapshot().getInitialIndex(); return GetNodeInfoResponse.create( diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index 3cfd578185..24873ff5e1 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -18,7 +18,6 @@ import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java b/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java index e3bcdabecd..7fe6e41833 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java @@ -17,7 +17,8 @@ * logic based on the type of the job. */ public class MilestonePrunerJobQueue implements JobQueue { - /** f + + /** * Holds the youngest (highest) milestone index that was successfully cleaned (gets updated when a job finishes). */ private int youngestFullyCleanedMilestoneIndex; diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index 838f418cf8..6f768abe35 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -127,7 +127,6 @@ public void process() throws TransactionPruningException { } getTransactionPruner().saveState(); - getSnapshot().setIndex(getCurrentIndex()); } } catch (TransactionPruningException e) { setStatus(TransactionPrunerJobStatus.FAILED); From b1d2d4d92cde6aa90582023fefe7d994502ff8af Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 16:15:13 +0100 Subject: [PATCH 49/63] Changed initial milestone to MilestoneVM.first() --- src/main/java/com/iota/iri/service/API.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 193470ea28..213875c03c 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -4,6 +4,7 @@ import com.iota.iri.conf.APIConfig; import com.iota.iri.controllers.AddressViewModel; import com.iota.iri.controllers.BundleViewModel; +import com.iota.iri.controllers.MilestoneViewModel; import com.iota.iri.controllers.TagViewModel; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.crypto.Curl; @@ -870,19 +871,11 @@ private AbstractResponse interruptAttachingToTangleStatement(){ * Returns information about this node. * * @return {@link com.iota.iri.service.dto.GetNodeInfoResponse} + * @throws Exception When we cant find the first milestone in the database **/ - private AbstractResponse getNodeInfoStatement(){ + private AbstractResponse getNodeInfoStatement() throws Exception{ String name = instance.configuration.isTestnet() ? IRI.TESTNET_NAME : IRI.MAINNET_NAME; - // Note: When the MilestonePrunerQueue is processing a job, it is possible that this index is out of date. - // Chances of this increase when a large localSnapshotsIntervalSynced is chosen. - // If pruning is off, return -1 - MilestonePrunerJobQueue milestoneJobQueue = instance.transactionPruner - .getJobQueueByQueueClass(MilestonePrunerJobQueue.class); - int initialIndex = milestoneJobQueue != null ? - milestoneJobQueue.getYoungestFullyCleanedMilestoneIndex() : - instance.snapshotProvider.getInitialSnapshot().getInitialIndex(); - return GetNodeInfoResponse.create( name, IRI.VERSION, @@ -898,7 +891,7 @@ private AbstractResponse getNodeInfoStatement(){ instance.snapshotProvider.getLatestSnapshot().getHash(), instance.snapshotProvider.getLatestSnapshot().getIndex(), - initialIndex, + MilestoneViewModel.first(instance.tangle).index(), instance.snapshotProvider.getLatestSnapshot().getInitialIndex(), instance.node.howManyNeighbors(), From 0cdaca902c2b9c90197d712edd691fb06345c3ab Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 16:20:35 +0100 Subject: [PATCH 50/63] removed unused functions for queue again --- .../transactionpruning/TransactionPruner.java | 8 -------- .../async/AsyncTransactionPruner.java | 12 ------------ .../async/MilestonePrunerJobQueue.java | 10 ---------- 3 files changed, 30 deletions(-) diff --git a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java index 0323a138f9..2bbe9a711d 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java @@ -21,14 +21,6 @@ public interface TransactionPruner { * @throws TransactionPruningException if anything goes wrong while adding the job */ void addJob(TransactionPrunerJob job) throws TransactionPruningException; - - /** - * Finds the specific JobQueue instance from the current queues. - * - * @param jobQueueType the class which extends of the JobQueue - * @return The JobQueue, or null if it does not exist - */ - T getJobQueueByQueueClass(Class jobQueueType); /** * This method executes all jobs that where added to the {@link TransactionPruner} through diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java index 24873ff5e1..dd7784b844 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/AsyncTransactionPruner.java @@ -165,18 +165,6 @@ public void addJob(TransactionPrunerJob job) throws TransactionPruningException saveState(); } - - /** - * {@inheritDoc} - * - * Gets the current job from its corresponding queue. - */ - @Override - public T getJobQueueByQueueClass(Class jobQueueType) { - return (T) jobQueues.values().stream(). - filter(jobQueue -> jobQueue.getClass().equals(jobQueueType)). - findFirst().orElse(null); - } /** * {@inheritDoc} diff --git a/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java b/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java index 7fe6e41833..afd741eb70 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/async/MilestonePrunerJobQueue.java @@ -165,14 +165,4 @@ private boolean adjustedRangeCoversNewMilestone(MilestonePrunerJob job, Mileston return true; } - - /** - * Gets the youngest cleaned milestone index. - * This is not guaranteed to be the current youngest in the database, since the current job could be making changes. - * - * @return The milestone index of the current youngest fully cleaned milestone in the provider - */ - public int getYoungestFullyCleanedMilestoneIndex() { - return youngestFullyCleanedMilestoneIndex; - } } From 50c94944ba5beaa875c7821c1e6e80d75f0ea505 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 17:30:37 +0200 Subject: [PATCH 51/63] Change the minimum value for LOCAL_SNAPSHOTS_PRUNING_DELAY (#1267) --- .../com/iota/iri/conf/BaseIotaConfig.java | 4 +-- .../java/com/iota/iri/conf/ConfigTest.java | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index a056a07d0e..0661110cf3 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -805,8 +805,8 @@ public interface Defaults { boolean LOCAL_SNAPSHOTS_ENABLED = true; boolean LOCAL_SNAPSHOTS_PRUNING_ENABLED = true; - int LOCAL_SNAPSHOTS_PRUNING_DELAY = 50000; - int LOCAL_SNAPSHOTS_PRUNING_DELAY_MIN = 40000; + int LOCAL_SNAPSHOTS_PRUNING_DELAY = 40000; + int LOCAL_SNAPSHOTS_PRUNING_DELAY_MIN = 10000; int LOCAL_SNAPSHOTS_INTERVAL_SYNCED = 10; int LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED = 1000; int LOCAL_SNAPSHOTS_DEPTH = 100; diff --git a/src/test/java/com/iota/iri/conf/ConfigTest.java b/src/test/java/com/iota/iri/conf/ConfigTest.java index 0f6b5d85e3..a4ef561dd3 100644 --- a/src/test/java/com/iota/iri/conf/ConfigTest.java +++ b/src/test/java/com/iota/iri/conf/ConfigTest.java @@ -107,6 +107,8 @@ public void testArgsParsingMainnet() { Assert.assertNotEquals("mwm", 4, iotaConfig.getMwm()); Assert.assertNotEquals("coo", iotaConfig.getCoordinator(), "TTTTTTTTT"); Assert.assertEquals("--testnet-no-coo-validation", false, iotaConfig.isDontValidateTestnetMilestoneSig()); + //Test default value + Assert.assertEquals("--local-snapshots-pruning-delay", 40000, iotaConfig.getLocalSnapshotsPruningDelay()); } @Test @@ -265,6 +267,32 @@ public void testInvalidIni() throws IOException { ConfigFactory.createFromFile(configFile, false); } + + @Test(expected = IllegalArgumentException.class) + public void pruningSnapshotDelayBelowMin() throws IOException { + String iniContent = new StringBuilder() + .append("[IRI]").append(System.lineSeparator()) + .append("LOCAL_SNAPSHOTS_PRUNING_DELAY = 9999") + .toString(); + try (Writer writer = new FileWriter(configFile)) { + writer.write(iniContent); + } + ConfigFactory.createFromFile(configFile, false); + } + + @Test + public void pruningSnapshotDelayIsMin() throws IOException { + String iniContent = new StringBuilder() + .append("[IRI]").append(System.lineSeparator()) + .append("LOCAL_SNAPSHOTS_PRUNING_DELAY = 10000") + .toString(); + try (Writer writer = new FileWriter(configFile)) { + writer.write(iniContent); + } + IotaConfig iotaConfig = ConfigFactory.createFromFile(configFile, false); + Assert.assertEquals("unexpected pruning delay", 10000, iotaConfig.getLocalSnapshotsPruningDelay()); + } + @Test public void backwardsIniCompatibilityTest() { Collection configNames = IotaUtils.getAllSetters(TestnetConfig.class) From d60600fac6a28eff9f5b386a557663c5e7e1aff5 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 17:07:07 +0100 Subject: [PATCH 52/63] removed unused import --- .../iota/iri/service/transactionpruning/TransactionPruner.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java index 2bbe9a711d..ed6c62de3a 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPruner.java @@ -1,7 +1,5 @@ package com.iota.iri.service.transactionpruning; -import com.iota.iri.service.transactionpruning.async.JobQueue; - /** * Represents the manager for the cleanup jobs that are issued by the * {@link com.iota.iri.service.snapshot.LocalSnapshotManager} in connection with local snapshots and eventually other From 1bc57d2828a21b254ec8e8d8091a06e4bd1390f0 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 17:59:56 +0100 Subject: [PATCH 53/63] added check for first milestone existance --- src/main/java/com/iota/iri/service/API.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 213875c03c..36b1a7203d 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -875,6 +875,7 @@ private AbstractResponse interruptAttachingToTangleStatement(){ **/ private AbstractResponse getNodeInfoStatement() throws Exception{ String name = instance.configuration.isTestnet() ? IRI.TESTNET_NAME : IRI.MAINNET_NAME; + MilestoneViewModel milestone = MilestoneViewModel.first(instance.tangle); return GetNodeInfoResponse.create( name, @@ -891,7 +892,7 @@ private AbstractResponse getNodeInfoStatement() throws Exception{ instance.snapshotProvider.getLatestSnapshot().getHash(), instance.snapshotProvider.getLatestSnapshot().getIndex(), - MilestoneViewModel.first(instance.tangle).index(), + milestone != null ? milestone.index() : -1, instance.snapshotProvider.getLatestSnapshot().getInitialIndex(), instance.node.howManyNeighbors(), From b9d751a9fbdff66162c70ac849a82046a57ce8e8 Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 18:03:03 +0100 Subject: [PATCH 54/63] Added lastSnapshottedMilestoneIndex to aloe --- python-regression/tests/features/machine1/1_api_tests.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/python-regression/tests/features/machine1/1_api_tests.feature b/python-regression/tests/features/machine1/1_api_tests.feature index c160d1d193..1b8d73fe7d 100644 --- a/python-regression/tests/features/machine1/1_api_tests.feature +++ b/python-regression/tests/features/machine1/1_api_tests.feature @@ -33,6 +33,7 @@ Feature: Test API calls on Machine 1 |latestSolidSubtangleMilestone | |latestSolidSubtangleMilestoneIndex | |milestoneStartIndex | + |lastSnapshottedMilestoneIndex | |neighbors | |packetsQueueSize | |time | From 13a3bad1ed8c73ece70409182c6087b04d7e57e5 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 19:38:09 +0200 Subject: [PATCH 55/63] bump version to 1.6.0-RC16 --- DOCKER.md | 6 +++--- pom.xml | 2 +- src/main/java/com/iota/iri/IRI.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index 8aba8d7d6a..bd8e83bf3c 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -2,13 +2,13 @@ Run the official iotaledger/iri container, passing the mandatory -p option: -```docker run iotaledger/iri:v1.6.0-RC15 -p 14265``` +```docker run iotaledger/iri:v1.6.0-RC16 -p 14265``` This will get your a running IRI with its API listening on port 14265, no neighbours and an empty database. The IRI Docker container by default expects data at /iri/data. Use the `-v` option of the `docker run` command to mount volumes so to have persistent data. You can also pass more command line options to the docker run command and those will be passed to IRI. If you want to use a iri.ini file with the docker container, supposing it's stored under /path/to/conf/iri.ini on your docker host, then pass `-v /path/to/conf:/iri/conf` and add -c /iri/conf/iri.ini as docker run arguments. So for example the `docker run` command above would become: -```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC15 -p 14265 -c /iri/conf/iri.ini``` +```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC16 -p 14265 -c /iri/conf/iri.ini``` Please refer to the IRI documentation for further command line options and iri.ini options. @@ -61,7 +61,7 @@ ExecStart=/usr/bin/docker run \ -p 14265:14265 \ -p 15600:15600 \ -p 14600:14600/udp \ -iotaledger/iri:v1.6.0-RC15 \ +iotaledger/iri:v1.6.0-RC16 \ -p 14265 \ --zmq-enabled \ --testnet diff --git a/pom.xml b/pom.xml index a6316fb2de..bf1f07ad50 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.iota iri - 1.6.0-RC15 + 1.6.0-RC16 IRI IOTA Reference Implementation diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index ca3ac017af..9cc2eda8fc 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -42,7 +42,7 @@ public class IRI { public static final String MAINNET_NAME = "IRI"; public static final String TESTNET_NAME = "IRI Testnet"; - public static final String VERSION = "1.6.0-RC15"; + public static final String VERSION = "1.6.0-RC16"; /** * The entry point of IRI. From 768210b3867cfb35d311a95819ea4dc6da9ea3ec Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 20:52:42 +0200 Subject: [PATCH 56/63] Don't start node if local snapshot is loaded but there is no spent addresses (#1266) --- src/main/java/com/iota/iri/Iota.java | 4 ++- .../snapshot/impl/SnapshotProviderImpl.java | 25 ++++++++++++++++--- .../SpentAddressesProvider.java | 9 +++++++ .../impl/SpentAddressesProviderImpl.java | 4 +-- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 63fb820f5c..9c3dc2e51a 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -196,9 +196,11 @@ public void init() throws Exception { } private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException { + //snapshot provider must be initialized first + //because we check whether spent addresses data exists + snapshotProvider.init(configuration); spentAddressesProvider.init(configuration); spentAddressesService.init(tangle, snapshotProvider, spentAddressesProvider); - snapshotProvider.init(configuration); snapshotService.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, configuration); if (localSnapshotManager != null) { localSnapshotManager.init(snapshotProvider, snapshotService, transactionPruner, configuration); diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java index 4ed64b1bc4..63f1a8b7e4 100644 --- a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java +++ b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotProviderImpl.java @@ -5,6 +5,8 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; import com.iota.iri.service.snapshot.*; +import com.iota.iri.service.spentaddresses.SpentAddressesException; +import com.iota.iri.service.spentaddresses.SpentAddressesProvider; import java.io.*; import java.nio.file.Files; @@ -94,7 +96,7 @@ public class SnapshotProviderImpl implements SnapshotProvider { * @return the initialized instance itself to allow chaining * */ - public SnapshotProviderImpl init(SnapshotConfig config) throws SnapshotException { + public SnapshotProviderImpl init(SnapshotConfig config) throws SnapshotException, SpentAddressesException { this.config = config; loadSnapshots(); @@ -181,7 +183,7 @@ public void shutdown() { * * @throws SnapshotException if anything goes wrong while loading the snapshots */ - private void loadSnapshots() throws SnapshotException { + private void loadSnapshots() throws SnapshotException, SpentAddressesException { initialSnapshot = loadLocalSnapshot(); if (initialSnapshot == null) { initialSnapshot = loadBuiltInSnapshot(); @@ -199,7 +201,7 @@ private void loadSnapshots() throws SnapshotException { * @return local snapshot of the node * @throws SnapshotException if local snapshot files exist but are malformed */ - private Snapshot loadLocalSnapshot() throws SnapshotException { + private Snapshot loadLocalSnapshot() throws SnapshotException, SpentAddressesException { if (config.getLocalSnapshotsEnabled()) { File localSnapshotFile = new File(config.getLocalSnapshotsBasePath() + ".snapshot.state"); File localSnapshotMetadDataFile = new File(config.getLocalSnapshotsBasePath() + ".snapshot.meta"); @@ -207,6 +209,8 @@ private Snapshot loadLocalSnapshot() throws SnapshotException { if (localSnapshotFile.exists() && localSnapshotFile.isFile() && localSnapshotMetadDataFile.exists() && localSnapshotMetadDataFile.isFile()) { + assertSpentAddressesDbExist(); + SnapshotState snapshotState = readSnapshotStatefromFile(localSnapshotFile.getAbsolutePath()); if (!snapshotState.hasCorrectSupply()) { throw new SnapshotException("the snapshot state file has an invalid supply"); @@ -226,6 +230,21 @@ private Snapshot loadLocalSnapshot() throws SnapshotException { return null; } + private void assertSpentAddressesDbExist() throws SpentAddressesException { + try { + File spentAddressFolder = new File(SpentAddressesProvider.SPENT_ADDRESSES_DB); + //If there is at least one file in the db the check should pass + if (Files.newDirectoryStream(spentAddressFolder.toPath(), "*.sst").iterator().hasNext()) { + return; + } + } + catch (IOException e){ + throw new SpentAddressesException("Can't load " + SpentAddressesProvider.SPENT_ADDRESSES_DB + " folder", e); + } + + throw new SpentAddressesException(SpentAddressesProvider.SPENT_ADDRESSES_DB + " folder has no sst files"); + } + /** * Loads the builtin snapshot (last global snapshot) that is embedded in the jar (if a different path is provided it * can also load from the disk). diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java index f37e9f6f47..f1cf8a4cb4 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java @@ -8,6 +8,15 @@ * Find, mark and store spent addresses */ public interface SpentAddressesProvider { + /** + * Folder where we store spent address data + */ + String SPENT_ADDRESSES_DB = "spent-addresses-db"; + + /** + * Folder where we store spent address logs + */ + String SPENT_ADDRESSES_LOG = "spent-addresses-log"; /** * Checks if this address hash has been spent from diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index c5a84587f4..6b827d96f5 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -39,8 +39,8 @@ public class SpentAddressesProviderImpl implements SpentAddressesProvider { * Creates a new instance of SpentAddressesProvider */ public SpentAddressesProviderImpl() { - this.rocksDBPersistenceProvider = new RocksDBPersistenceProvider("spent-addresses-db", - "spent-addresses-log", 1000, + this.rocksDBPersistenceProvider = new RocksDBPersistenceProvider(SPENT_ADDRESSES_DB, + SPENT_ADDRESSES_LOG, 1000, new HashMap>(1) {{put("spent-addresses", SpentAddress.class);}}, null); } From aee90cc6e5429f653527be5a8693612318e961fe Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 20:31:11 +0100 Subject: [PATCH 57/63] spacing --- python-regression/tests/features/machine1/1_api_tests.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-regression/tests/features/machine1/1_api_tests.feature b/python-regression/tests/features/machine1/1_api_tests.feature index 1b8d73fe7d..9926868685 100644 --- a/python-regression/tests/features/machine1/1_api_tests.feature +++ b/python-regression/tests/features/machine1/1_api_tests.feature @@ -33,7 +33,7 @@ Feature: Test API calls on Machine 1 |latestSolidSubtangleMilestone | |latestSolidSubtangleMilestoneIndex | |milestoneStartIndex | - |lastSnapshottedMilestoneIndex | + |lastSnapshottedMilestoneIndex | |neighbors | |packetsQueueSize | |time | From 0984753311d6c79e8364fb9f8ecd5044c1f2b63f Mon Sep 17 00:00:00 2001 From: Brord van Wierst Date: Thu, 10 Jan 2019 20:32:04 +0100 Subject: [PATCH 58/63] spacing2 --- python-regression/tests/features/machine1/1_api_tests.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-regression/tests/features/machine1/1_api_tests.feature b/python-regression/tests/features/machine1/1_api_tests.feature index 9926868685..135210f904 100644 --- a/python-regression/tests/features/machine1/1_api_tests.feature +++ b/python-regression/tests/features/machine1/1_api_tests.feature @@ -33,7 +33,7 @@ Feature: Test API calls on Machine 1 |latestSolidSubtangleMilestone | |latestSolidSubtangleMilestoneIndex | |milestoneStartIndex | - |lastSnapshottedMilestoneIndex | + |lastSnapshottedMilestoneIndex | |neighbors | |packetsQueueSize | |time | From 0883613705a00437ee7125b394249b6a6b38ed77 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 21:58:47 +0200 Subject: [PATCH 59/63] bump to 1.6.0-RC17 --- DOCKER.md | 6 +++--- pom.xml | 2 +- src/main/java/com/iota/iri/IRI.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index bd8e83bf3c..c0f9156b27 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -2,13 +2,13 @@ Run the official iotaledger/iri container, passing the mandatory -p option: -```docker run iotaledger/iri:v1.6.0-RC16 -p 14265``` +```docker run iotaledger/iri:v1.6.0-RC17 -p 14265``` This will get your a running IRI with its API listening on port 14265, no neighbours and an empty database. The IRI Docker container by default expects data at /iri/data. Use the `-v` option of the `docker run` command to mount volumes so to have persistent data. You can also pass more command line options to the docker run command and those will be passed to IRI. If you want to use a iri.ini file with the docker container, supposing it's stored under /path/to/conf/iri.ini on your docker host, then pass `-v /path/to/conf:/iri/conf` and add -c /iri/conf/iri.ini as docker run arguments. So for example the `docker run` command above would become: -```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC16 -p 14265 -c /iri/conf/iri.ini``` +```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC17 -p 14265 -c /iri/conf/iri.ini``` Please refer to the IRI documentation for further command line options and iri.ini options. @@ -61,7 +61,7 @@ ExecStart=/usr/bin/docker run \ -p 14265:14265 \ -p 15600:15600 \ -p 14600:14600/udp \ -iotaledger/iri:v1.6.0-RC16 \ +iotaledger/iri:v1.6.0-RC17 \ -p 14265 \ --zmq-enabled \ --testnet diff --git a/pom.xml b/pom.xml index bf1f07ad50..3410c19134 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.iota iri - 1.6.0-RC16 + 1.6.0-RC17 IRI IOTA Reference Implementation diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index 9cc2eda8fc..9e6b6497cc 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -42,7 +42,7 @@ public class IRI { public static final String MAINNET_NAME = "IRI"; public static final String TESTNET_NAME = "IRI Testnet"; - public static final String VERSION = "1.6.0-RC16"; + public static final String VERSION = "1.6.0-RC17"; /** * The entry point of IRI. From 2b497ba30738c74802eb5d1e0e84089e3b2c6a37 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 22:04:24 +0200 Subject: [PATCH 60/63] Don't throw on missed spent address. Persist it and warn (#1271) * refactor wasAddressSpentFrom * refactor exists() * log warning rather than throwing an error --- .../iri/model/persistables/SpentAddress.java | 6 +++++- .../impl/SpentAddressesProviderImpl.java | 2 +- .../impl/SpentAddressesServiceImpl.java | 15 +++------------ .../jobs/MilestonePrunerJob.java | 16 ++++++++++++---- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/iota/iri/model/persistables/SpentAddress.java b/src/main/java/com/iota/iri/model/persistables/SpentAddress.java index 99c6c8d271..3c2372d474 100644 --- a/src/main/java/com/iota/iri/model/persistables/SpentAddress.java +++ b/src/main/java/com/iota/iri/model/persistables/SpentAddress.java @@ -3,7 +3,7 @@ import com.iota.iri.storage.Persistable; public class SpentAddress implements Persistable { - public boolean exists = false; + private boolean exists = false; @Override public byte[] bytes() { @@ -28,4 +28,8 @@ public void readMetadata(byte[] bytes) { public boolean merge() { return false; } + + public boolean exists() { + return exists; + } } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 6b827d96f5..1ae53506fc 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -90,7 +90,7 @@ private void readSpentAddressesFromStream(InputStream in) throws SpentAddressesE @Override public boolean containsAddress(Hash addressHash) throws SpentAddressesException { try { - return ((SpentAddress) rocksDBPersistenceProvider.get(SpentAddress.class, addressHash)).exists; + return ((SpentAddress) rocksDBPersistenceProvider.get(SpentAddress.class, addressHash)).exists(); } catch (Exception e) { throw new SpentAddressesException(e); } diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java index a18a1fdead..cccf794dd5 100644 --- a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java +++ b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java @@ -61,19 +61,10 @@ public boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesExcept try { Set hashes = AddressViewModel.load(tangle, addressHash).getHashes(); for (Hash hash : hashes) { - final TransactionViewModel tx = TransactionViewModel.fromHash(tangle, hash); + TransactionViewModel tx = TransactionViewModel.fromHash(tangle, hash); // Check for spending transactions - if (tx.value() < 0) { - // Transaction is confirmed - if (tx.snapshotIndex() != 0) { - return true; - } - - // Transaction is pending - Optional tail = tailFinder.findTailFromTx(tx); - if (tail.isPresent()) { - return isBundleValid(tail.get()); - } + if (wasTransactionSpentFrom(tx)) { + return true; } } } catch (Exception e) { diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index 6f768abe35..86a0649961 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -14,8 +14,12 @@ import com.iota.iri.utils.dag.DAGHelper; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Represents a cleanup job for {@link com.iota.iri.service.transactionpruning.TransactionPruner}s that removes * milestones and all of their directly and indirectly referenced transactions (and the orphaned subtangles branching @@ -26,6 +30,9 @@ * time, persisting the progress after each step. */ public class MilestonePrunerJob extends AbstractTransactionPrunerJob { + + private static final Logger log = LoggerFactory.getLogger(MilestonePrunerJob.class); + /** * Holds the milestone index where this job starts cleaning up. */ @@ -266,11 +273,12 @@ private void cleanupMilestoneTransactions() throws TransactionPruningException { approvedTransaction -> { if (approvedTransaction.value() < 0 && !spentAddressesService.wasAddressSpentFrom(approvedTransaction.getAddressHash())) { - throw new SpentAddressesException( - "Pruned spend transaction did not have its spent address recorded"); - } else { - elementsToDelete.add(new Pair<>(approvedTransaction.getHash(), Transaction.class)); + log.warn("Pruned spend transaction " + approvedTransaction.getHash() + + " did not have its spent address recorded. Persisting it now"); + spentAddressesService + .persistSpentAddresses(Collections.singletonList(approvedTransaction)); } + elementsToDelete.add(new Pair<>(approvedTransaction.getHash(), Transaction.class)); }); } From 8cd3c3efad157cc25ac87cfb32ffc705311bbc2e Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Thu, 10 Jan 2019 22:05:35 +0200 Subject: [PATCH 61/63] bump to 1.6.0-RC18 --- pom.xml | 2 +- src/main/java/com/iota/iri/IRI.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 3410c19134..5f0b61a8cf 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.iota iri - 1.6.0-RC17 + 1.6.0-RC18 IRI IOTA Reference Implementation diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index 9e6b6497cc..53a7da5429 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -42,7 +42,7 @@ public class IRI { public static final String MAINNET_NAME = "IRI"; public static final String TESTNET_NAME = "IRI Testnet"; - public static final String VERSION = "1.6.0-RC17"; + public static final String VERSION = "1.6.0-RC18"; /** * The entry point of IRI. From 952026bf7847fc9eed56fb547190219223d91340 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Fri, 11 Jan 2019 17:28:25 +0200 Subject: [PATCH 62/63] minor import and capitalization fixi --- src/main/java/com/iota/iri/IRI.java | 2 +- .../iri/service/transactionpruning/jobs/MilestonePrunerJob.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index 53a7da5429..c716ea2253 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -53,7 +53,7 @@ public class IRI { * @throws Exception If we fail to start the IRI launcher. */ public static void main(String[] args) throws Exception { - // Logging is configured first before any references to Logger or LoggerFactory. + // Logging is configured first before ANY references to Logger or LoggerFactory. // Any public method or field accessors needed in IRI should be put in IRI and then delegate to IRILauncher. // That ensures that future code does not need to know about this setup. configureLogging(); diff --git a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java index 86a0649961..b795a3de83 100644 --- a/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java +++ b/src/main/java/com/iota/iri/service/transactionpruning/jobs/MilestonePrunerJob.java @@ -5,7 +5,6 @@ import com.iota.iri.model.IntegerIndex; import com.iota.iri.model.persistables.Milestone; import com.iota.iri.model.persistables.Transaction; -import com.iota.iri.service.spentaddresses.SpentAddressesException; import com.iota.iri.service.transactionpruning.TransactionPrunerJobStatus; import com.iota.iri.service.transactionpruning.TransactionPruningException; import com.iota.iri.storage.Indexable; From f6e84bbc3fb27ec42f7302908c81cdc40711b34a Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Fri, 11 Jan 2019 17:44:42 +0200 Subject: [PATCH 63/63] bump version to release v1.6.0 --- DOCKER.md | 6 +++--- pom.xml | 2 +- src/main/java/com/iota/iri/IRI.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index c0f9156b27..3fbf4ea7f7 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -2,13 +2,13 @@ Run the official iotaledger/iri container, passing the mandatory -p option: -```docker run iotaledger/iri:v1.6.0-RC17 -p 14265``` +```docker run iotaledger/iri:v1.6.0-RELEASE -p 14265``` This will get your a running IRI with its API listening on port 14265, no neighbours and an empty database. The IRI Docker container by default expects data at /iri/data. Use the `-v` option of the `docker run` command to mount volumes so to have persistent data. You can also pass more command line options to the docker run command and those will be passed to IRI. If you want to use a iri.ini file with the docker container, supposing it's stored under /path/to/conf/iri.ini on your docker host, then pass `-v /path/to/conf:/iri/conf` and add -c /iri/conf/iri.ini as docker run arguments. So for example the `docker run` command above would become: -```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RC17 -p 14265 -c /iri/conf/iri.ini``` +```docker run -v /path/to/conf:/iri/conf -v /path/to/data:/iri/data iotaledger/iri:v1.6.0-RELEASE -p 14265 -c /iri/conf/iri.ini``` Please refer to the IRI documentation for further command line options and iri.ini options. @@ -61,7 +61,7 @@ ExecStart=/usr/bin/docker run \ -p 14265:14265 \ -p 15600:15600 \ -p 14600:14600/udp \ -iotaledger/iri:v1.6.0-RC17 \ +iotaledger/iri:v1.6.0-RELEASE \ -p 14265 \ --zmq-enabled \ --testnet diff --git a/pom.xml b/pom.xml index 5f0b61a8cf..955c127ad0 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.iota iri - 1.6.0-RC18 + 1.6.0-RELEASE IRI IOTA Reference Implementation diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java index c716ea2253..c09a7aa94c 100644 --- a/src/main/java/com/iota/iri/IRI.java +++ b/src/main/java/com/iota/iri/IRI.java @@ -42,7 +42,7 @@ public class IRI { public static final String MAINNET_NAME = "IRI"; public static final String TESTNET_NAME = "IRI Testnet"; - public static final String VERSION = "1.6.0-RC18"; + public static final String VERSION = "1.6.0-RELEASE"; /** * The entry point of IRI.