From 0b8a6bde7f615469662d62a833e9c08d5d9b043a Mon Sep 17 00:00:00 2001 From: olaf Date: Wed, 7 Aug 2019 14:47:59 +0200 Subject: [PATCH 01/26] Counted transaction model for selective permanode storage capabilities based on the current TxModel and rocksdb persistence layer. --- .../com/iota/iri/conf/BaseIotaConfig.java | 101 ++++++++++++++++++ .../java/com/iota/iri/conf/IotaConfig.java | 3 +- .../java/com/iota/iri/conf/PermaDBConfig.java | 69 ++++++++++++ .../persistables/CountedTransaction.java | 54 ++++++++++ .../java/com/iota/iri/storage/Tangle.java | 25 +++-- .../persistables/CountedTransactionTest.java | 57 ++++++++++ .../PermaRocksDBPersentenceProviderTest.java | 76 +++++++++++++ .../RocksDBPersistenceProviderTest.java | 11 +- 8 files changed, 384 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/iota/iri/conf/PermaDBConfig.java create mode 100644 src/main/java/com/iota/iri/model/persistables/CountedTransaction.java create mode 100644 src/test/java/com/iota/iri/model/persistables/CountedTransactionTest.java create mode 100644 src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index 5da064717b..e6a6bf187f 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -68,6 +68,15 @@ public abstract class BaseIotaConfig implements IotaConfig { protected boolean revalidate = Defaults.REVALIDATE; protected boolean rescanDb = Defaults.RESCAN_DB; + //PERMADB + protected boolean permaDbEnabled = Defaults.PERMADB_ENABLED; + protected String permaDbPath = Defaults.PERMADB_PATH; + protected String permaDbLogPath = Defaults.PERMADB_LOG_PATH; + protected int permaDbCacheSize = Defaults.PERMADB_CACHE_SIZE; //KB + protected String permaMainDb = Defaults.PERMAMAIN_DB; + protected boolean permaRevalidate = Defaults.PERMAREVALIDATE; + protected boolean permaRescanDb = Defaults.PERMARESCAN_DB; + //Protocol protected double pSendMilestone = Defaults.P_SEND_MILESTONE; @@ -381,6 +390,89 @@ protected void setIxiDir(String ixiDir) { this.ixiDir = ixiDir; } + + + + //------- PERMADB ------------- + + + @Override + public boolean isSelectivePermaEnabled() { return this.permaDbEnabled;} + + @JsonProperty + @Parameter(names = {"--perma-enabled"}, description = PermaDBConfig.Descriptions.PERMADB_ENABLED, arity = 1) + protected void setPermaDbEnabled(boolean permaDbEnabled) { + this.permaDbEnabled = permaDbEnabled; + } + + @Override + public String getPermaDbPath() { + return permaDbPath; + } + + @JsonProperty + @Parameter(names = {"--perma-db-path"}, description = PermaDBConfig.Descriptions.PERMADB_PATH) + protected void setPermaDbPath(String permaDbPath) { + this.permaDbPath = permaDbPath; + } + + @Override + public String getPermaDbLogPath() { + return dbLogPath; + } + + @JsonProperty + @Parameter(names = {"--perma-db-log-path"}, description = PermaDBConfig.Descriptions.PERMADB_LOG_PATH) + protected void setPermaDbLogPath(String permaDbLogPath) { + this.permaDbLogPath = permaDbLogPath; + } + + @Override + public int getPermaDbCacheSize() { + return permaDbCacheSize; + } + + @JsonProperty + @Parameter(names = {"--perma-db-cache-size"}, description = PermaDBConfig.Descriptions.PERMADB_CACHE_SIZE) + protected void setPermaDbCacheSize(int permaDbCacheSize) { + this.permaDbCacheSize = permaDbCacheSize; + } + + @Override + public String getPermaMainDb() { + return permaMainDb; + } + + @JsonProperty + @Parameter(names = {"--perma-db"}, description = PermaDBConfig.Descriptions.PERMAMAIN_DB) + protected void setPermaMainDb(String permaMainDb) { + this.permaMainDb = permaMainDb; + } + + @Override + public boolean permaIsRevalidate() { + return permaRevalidate; + } + + @JsonProperty + @Parameter(names = {"--perma-revalidate"}, description = PermaDBConfig.Descriptions.PERMAREVALIDATE, arity = 1) + protected void setPermaRevalidate(boolean permaRevalidate) { + this.permaRevalidate = permaRevalidate; + } + + @Override + public boolean permaIsRescanDb() { + return this.permaRescanDb; + } + + @JsonProperty + @Parameter(names = {"--perma-rescan"}, description = PermaDBConfig.Descriptions.PERMARESCAN_DB, arity = 1) + protected void setPermaRescanDb(boolean permaRescanDb) { + this.permaRescanDb = permaRescanDb; + } + + + //----------------------------- @Override public String getDbPath() { return dbPath; @@ -855,6 +947,15 @@ public interface Defaults { boolean REVALIDATE = false; boolean RESCAN_DB = false; + //PERMADB + boolean PERMADB_ENABLED = false; + String PERMADB_PATH = "mainnetpermadb"; + String PERMADB_LOG_PATH = "mainnetpermadb.log"; + int PERMADB_CACHE_SIZE = 100_000; + String PERMAMAIN_DB = "rocksdb"; + boolean PERMAREVALIDATE = false; + boolean PERMARESCAN_DB = false; + //Protocol double P_SEND_MILESTONE = 0.02d; int MWM = 14; diff --git a/src/main/java/com/iota/iri/conf/IotaConfig.java b/src/main/java/com/iota/iri/conf/IotaConfig.java index a6e0923f64..2045e46c41 100644 --- a/src/main/java/com/iota/iri/conf/IotaConfig.java +++ b/src/main/java/com/iota/iri/conf/IotaConfig.java @@ -10,7 +10,8 @@ * In charge of how we parse the configuration from given inputs. */ public interface IotaConfig extends APIConfig, NodeConfig, - IXIConfig, DbConfig, ConsensusConfig, ZMQConfig, TipSelConfig, PearlDiverConfig, SolidificationConfig { + IXIConfig, DbConfig, PermaDBConfig, + ConsensusConfig, ZMQConfig, TipSelConfig, PearlDiverConfig, SolidificationConfig { File CONFIG_FILE = new File("iota.ini"); /** diff --git a/src/main/java/com/iota/iri/conf/PermaDBConfig.java b/src/main/java/com/iota/iri/conf/PermaDBConfig.java new file mode 100644 index 0000000000..01a623fae8 --- /dev/null +++ b/src/main/java/com/iota/iri/conf/PermaDBConfig.java @@ -0,0 +1,69 @@ +package com.iota.iri.conf; + +/** + * Configurations for tangle database. + */ +public interface PermaDBConfig extends Config { + + + /** + * Default Value: {@value BaseIotaConfig.Defaults#PERMADB_ENABLED} + * + * @return {@value PermaDBConfig.Descriptions#PERMADB_ENABLED} + */ + boolean isSelectivePermaEnabled(); + + /** + * Default Value: {@value BaseIotaConfig.Defaults#PERMADB_PATH} + * + * @return {@value PermaDBConfig.Descriptions#PERMADB_PATH} + */ + String getPermaDbPath(); + + /** + * Default Value: {@value BaseIotaConfig.Defaults#PERMADB_LOG_PATH} + * + * @return {@value PermaDBConfig.Descriptions#PERMADB_LOG_PATH} + */ + String getPermaDbLogPath(); + + /** + * Default Value: {@value BaseIotaConfig.Defaults#PERMADB_CACHE_SIZE} + * + * @return {@value PermaDBConfig.Descriptions#PERMADB_CACHE_SIZE} + */ + int getPermaDbCacheSize(); + + /** + * Default Value: {@value BaseIotaConfig.Defaults#PERMAMAIN_DB} + * + * @return {@value PermaDBConfig.Descriptions#PERMAMAIN_DB} + */ + String getPermaMainDb(); + + /** + * Default Value: {@value BaseIotaConfig.Defaults#PERMAREVALIDATE} + * + * @return {@value PermaDBConfig.Descriptions#PERMAREVALIDATE} + */ + boolean permaIsRevalidate(); + + /** + * Default Value: {@value BaseIotaConfig.Defaults#PERMARESCAN_DB} + * + * @return {@value PermaDBConfig.Descriptions#PERMARESCAN_DB} + */ + boolean permaIsRescanDb(); + + interface Descriptions { + + String PERMADB_PATH = "The folder where the DB saves its data."; + String PERMADB_LOG_PATH = "The folder where the DB logs info"; + String PERMADB_CACHE_SIZE = "The size of the DB cache in KB"; + String PERMAMAIN_DB = "The DB engine used to store the transactions. Currently only RocksDB is supported."; + String PERMAREVALIDATE = "Reload from the db data about confirmed transaction (milestones), state of the ledger, " + + "and transaction metadata."; + String PERMARESCAN_DB = "Rescan all transaction metadata (Approvees, Bundles, and Tags)"; + String PERMADB_ENABLED = "Enables secondary permanent storage"; + } +} diff --git a/src/main/java/com/iota/iri/model/persistables/CountedTransaction.java b/src/main/java/com/iota/iri/model/persistables/CountedTransaction.java new file mode 100644 index 0000000000..51cec8af50 --- /dev/null +++ b/src/main/java/com/iota/iri/model/persistables/CountedTransaction.java @@ -0,0 +1,54 @@ +package com.iota.iri.model.persistables; + +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.model.HashFactory; +import com.iota.iri.utils.Serializer; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +public class CountedTransaction extends Transaction { + + public static int counterSize = Long.BYTES; + public long storageReferenceCounter = 0l; + + public static CountedTransaction fromTransaction(Transaction tx, long storageReferenceCounter){ + CountedTransaction toReturn = new CountedTransaction(); + toReturn.bytes = tx.bytes(); + byte[] metadataBuffer = tx.metadata(); + ByteBuffer bb = ByteBuffer.allocate(metadataBuffer.length + counterSize); + bb.putLong(storageReferenceCounter); + bb.put(metadataBuffer); + toReturn.readMetadata(bb.array()); + + return toReturn; + } + + + @Override + public byte[] metadata() { + byte[] metadataBuffer = super.metadata(); + ByteBuffer bb = ByteBuffer.allocate(metadataBuffer.length + counterSize); + bb.putLong(storageReferenceCounter); + bb.put(metadataBuffer); + return bb.array(); + } + + + @Override + public void readMetadata(byte[] bytes) { + if(bytes != null) { + byte[] counterBytes = new byte[counterSize]; + System.arraycopy(bytes, 0, counterBytes, 0, counterSize); + storageReferenceCounter = Serializer.getLong(counterBytes, 0); + byte[] metadataBytes = new byte[bytes.length - counterSize]; + System.arraycopy(bytes, counterSize, metadataBytes, 0, metadataBytes.length); + + + super.readMetadata(metadataBytes); + } + + + } +} diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index b88ab25c0f..3b1679031f 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -2,13 +2,7 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.StateDiff; -import com.iota.iri.model.persistables.Address; -import com.iota.iri.model.persistables.Approvee; -import com.iota.iri.model.persistables.Bundle; -import com.iota.iri.model.persistables.Milestone; -import com.iota.iri.model.persistables.ObsoleteTag; -import com.iota.iri.model.persistables.Tag; -import com.iota.iri.model.persistables.Transaction; +import com.iota.iri.model.persistables.*; import com.iota.iri.utils.Pair; import java.util.*; @@ -28,6 +22,7 @@ public class Tangle { public static final Map> COLUMN_FAMILIES = new LinkedHashMap>() {{ put("transaction", Transaction.class); + put("counted-transaction", CountedTransaction.class); put("milestone", Milestone.class); put("stateDiff", StateDiff.class); put("address", Address.class); @@ -40,6 +35,22 @@ public class Tangle { public static final Map.Entry> METADATA_COLUMN_FAMILY = new AbstractMap.SimpleImmutableEntry<>("transaction-metadata", Transaction.class); + public static final Map.Entry> COUNTED_METADATA_COLUMN_FAMILY = + new AbstractMap.SimpleImmutableEntry<>("counted-transaction-metadata", CountedTransaction.class); + +// = +// new LinkedHashMap>() {{ +// put("transaction", Transaction.class); +// put("counted-transaction-metadata", CountedTransaction.class); +// put("milestone", Milestone.class); +// put("stateDiff", StateDiff.class); +// put("address", Address.class); +// put("approvee", Approvee.class); +// put("bundle", Bundle.class); +// put("obsoleteTag", ObsoleteTag.class); +// put("tag", Tag.class); +// }}; + private final List persistenceProviders = new ArrayList<>(); private final List messageQueueProviders = new ArrayList<>(); diff --git a/src/test/java/com/iota/iri/model/persistables/CountedTransactionTest.java b/src/test/java/com/iota/iri/model/persistables/CountedTransactionTest.java new file mode 100644 index 0000000000..4f87544677 --- /dev/null +++ b/src/test/java/com/iota/iri/model/persistables/CountedTransactionTest.java @@ -0,0 +1,57 @@ +package com.iota.iri.model.persistables; + +import com.iota.iri.TransactionTestUtils; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.utils.Converter; +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class CountedTransactionTest { + + + + @Test + public void testBytes() { + Transaction t = TransactionTestUtils.getTransaction(); + + Transaction newtx = new Transaction(); + newtx.read(t.bytes()); + newtx.readMetadata(t.metadata()); + CountedTransaction ctx1 = CountedTransaction.fromTransaction(newtx, 5); + + CountedTransaction ctx2 = new CountedTransaction(); + ctx2.read(ctx1.bytes()); + ctx2.readMetadata(ctx1.metadata()); + + + assertArrayEquals("metadata should be the same in the copy", ctx1.metadata(), ctx2.metadata()); + assertArrayEquals("bytes should be the same in the copy", newtx.bytes(), ctx2.bytes()); + } + +// @Test +// public void fromTrits() { +// byte[] trits = TransactionTestUtils.getTransactionTrits(); +// byte[] bytes = Converter.allocateBytesForTrits(trits.length); +// Converter.bytes(trits, bytes); +// +// TransactionViewModel tvm = new TransactionViewModel(trits, Hash.NULL_HASH); +// tvm.getAddressHash(); +// tvm.getTrunkTransactionHash(); +// tvm.getBranchTransactionHash(); +// tvm.getBundleHash(); +// tvm.getTagValue(); +// tvm.getObsoleteTagValue(); +// tvm.setAttachmentData(); +// tvm.setMetadata(); +// +// assertArrayEquals("bytes in the TVM should be unmodified", tvm.getBytes(), bytes); +// +// Transaction tvmTransaction = tvm.getTransaction(); +// +// assertEquals("branch in transaction should be the same as in the tvm", tvmTransaction.branch, tvm.getTransaction().branch); +// } + +} diff --git a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java new file mode 100644 index 0000000000..112b4649c6 --- /dev/null +++ b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java @@ -0,0 +1,76 @@ +package com.iota.iri.storage.rocksDB; + +import com.iota.iri.model.IntegerIndex; +import com.iota.iri.model.persistables.Transaction; +import com.iota.iri.storage.Indexable; +import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.Tangle; +import com.iota.iri.utils.Pair; +import org.apache.commons.io.FileUtils; +import org.junit.*; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class PermaRocksDBPersentenceProviderTest { + private static RocksDBPersistenceProvider rocksDBPersistenceProvider; + private static String dbPath = "tmpdb", dbLogPath = "tmplogs"; + + @BeforeClass + public static void setUpDb() throws Exception { + rocksDBPersistenceProvider = new RocksDBPersistenceProvider( + dbPath, dbLogPath,1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); + rocksDBPersistenceProvider.init(); + } + + @AfterClass + public static void destroyDb() { + rocksDBPersistenceProvider.shutdown(); + FileUtils.deleteQuietly(new File(dbPath)); + FileUtils.deleteQuietly(new File(dbLogPath)); + rocksDBPersistenceProvider = null; + } + + @Before + public void setUp() throws Exception { + rocksDBPersistenceProvider.clear(Transaction.class); + } + + @Test + public void testDeleteBatch() throws Exception { + Persistable tx = new Transaction(); + byte[] bytes = new byte[Transaction.SIZE]; + Arrays.fill(bytes, (byte) 1); + tx.read(bytes); + tx.readMetadata(bytes); + List> models = IntStream.range(1, 1000) + .mapToObj(i -> new Pair<>((Indexable) new IntegerIndex(i), tx)) + .collect(Collectors.toList()); + + rocksDBPersistenceProvider.saveBatch(models); + + List>> modelsToDelete = models.stream() + .filter(entry -> ((IntegerIndex) entry.low).getValue() < 900) + .map(entry -> new Pair<>(entry.low, entry.hi.getClass())) + .collect(Collectors.toList()); + + rocksDBPersistenceProvider.deleteBatch(modelsToDelete); + + for (Pair> model : modelsToDelete) { + Assert.assertNull("value at index " + ((IntegerIndex) model.low).getValue() + " should be deleted", + rocksDBPersistenceProvider.get(model.hi, model.low).bytes()); + } + + List indexes = IntStream.range(900, 1000) + .mapToObj(i -> new IntegerIndex(i)) + .collect(Collectors.toList()); + + for (IntegerIndex index : indexes) { + Assert.assertArrayEquals("saved bytes are not as expected in index " + index.getValue(), tx.bytes(), + rocksDBPersistenceProvider.get(Transaction.class, index).bytes()); + } + } +} diff --git a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java index b0f9b49b32..ea7cdccc96 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java @@ -1,6 +1,7 @@ package com.iota.iri.storage.rocksDB; import com.iota.iri.model.IntegerIndex; +import com.iota.iri.model.persistables.CountedTransaction; import com.iota.iri.model.persistables.Transaction; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; @@ -25,7 +26,7 @@ public class RocksDBPersistenceProviderTest { @BeforeClass public static void setUpDb() throws Exception { rocksDBPersistenceProvider = new RocksDBPersistenceProvider( - dbPath, dbLogPath,1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); + dbPath, dbLogPath,1000, Tangle.COLUMN_FAMILIES, Tangle.COUNTED_METADATA_COLUMN_FAMILY); rocksDBPersistenceProvider.init(); } @@ -44,13 +45,15 @@ public void setUp() throws Exception { @Test public void testDeleteBatch() throws Exception { - Persistable tx = new Transaction(); + Transaction tx = new Transaction(); byte[] bytes = new byte[Transaction.SIZE]; Arrays.fill(bytes, (byte) 1); tx.read(bytes); tx.readMetadata(bytes); + + List> models = IntStream.range(1, 1000) - .mapToObj(i -> new Pair<>((Indexable) new IntegerIndex(i), tx)) + .mapToObj(i -> new Pair<>((Indexable) new IntegerIndex(i), (Persistable)CountedTransaction.fromTransaction(tx,i))) .collect(Collectors.toList()); rocksDBPersistenceProvider.saveBatch(models); @@ -73,7 +76,7 @@ public void testDeleteBatch() throws Exception { for (IntegerIndex index : indexes) { Assert.assertArrayEquals("saved bytes are not as expected in index " + index.getValue(), tx.bytes(), - rocksDBPersistenceProvider.get(Transaction.class, index).bytes()); + rocksDBPersistenceProvider.get(CountedTransaction.class, index).bytes()); } } } \ No newline at end of file From 27fa8219e79eba7b8c46292735951c116beecd32 Mon Sep 17 00:00:00 2001 From: olaf Date: Tue, 20 Aug 2019 19:05:30 +0200 Subject: [PATCH 02/26] - Removed counted transaction model - Introduced an interface for permanent storage. - Added a permanent storage PersitenceProvider - Added mergeTwo() function to genericly handle class independant merging. - Changed Tangle.load() to handle multiple persistence providers and handle with mergable results (like searching addresses in two persistence providers) - Tangle.load() didn't handle this because persistenceProvider.get never returned null, only a new (empty) instance of the requested object. The bytes() wasnt uniform in implementation and would for example throw an exception on Milestone, the MilestoneViewModel did the null check. Therefore the introduction of isEmpty() for Persistables. --- src/main/java/com/iota/iri/Iota.java | 17 + .../com/iota/iri/conf/BaseIotaConfig.java | 10 +- .../java/com/iota/iri/model/StateDiff.java | 10 + .../persistables/CountedTransaction.java | 54 -- .../iota/iri/model/persistables/Hashes.java | 17 + .../iri/model/persistables/Milestone.java | 11 + .../iri/model/persistables/SpentAddress.java | 11 + .../iri/model/persistables/Transaction.java | 10 + .../CountedPermanentPersitenceProvider.java | 26 + .../com/iota/iri/storage/Persistable.java | 5 + .../java/com/iota/iri/storage/Tangle.java | 44 +- .../iri/storage/rocksDB/RocksDBPPPImpl.java | 555 ++++++++++++++++++ .../persistables/CountedTransactionTest.java | 57 -- .../PermaRocksDBPersentenceProviderTest.java | 111 ++-- .../RocksDBPersistenceProviderTest.java | 7 +- 15 files changed, 758 insertions(+), 187 deletions(-) delete mode 100644 src/main/java/com/iota/iri/model/persistables/CountedTransaction.java create mode 100644 src/main/java/com/iota/iri/storage/CountedPermanentPersitenceProvider.java create mode 100644 src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java delete mode 100644 src/test/java/com/iota/iri/model/persistables/CountedTransactionTest.java diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 64d39bbcc0..7a30ebb431 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Map; +import com.iota.iri.storage.rocksDB.RocksDBPPPImpl; import org.apache.commons.lang3.NotImplementedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -323,6 +324,22 @@ private void initializeTangle() { if (configuration.isZmqEnabled()) { tangle.addMessageQueueProvider(new ZmqMessageQueueProvider(configuration)); } + + if(configuration.isSelectivePermaEnabled()){ + switch (configuration.getPermaMainDb()) { + case "rocksdb": { + tangle.addPersistenceProvider(new RocksDBPPPImpl( + configuration.getPermaDbPath(), + configuration.getPermaDbLogPath(), + configuration.getPermaDbCacheSize()) + ); + break; + } + default: { + throw new NotImplementedException("No such database type."); + } + } + } } /** diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index e6a6bf187f..832e1c612c 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -400,9 +400,9 @@ protected void setIxiDir(String ixiDir) { public boolean isSelectivePermaEnabled() { return this.permaDbEnabled;} @JsonProperty - @Parameter(names = {"--perma-enabled"}, description = PermaDBConfig.Descriptions.PERMADB_ENABLED, arity = 1) - protected void setPermaDbEnabled(boolean permaDbEnabled) { - this.permaDbEnabled = permaDbEnabled; + @Parameter(names = {"--permadb-enabled"}, description = PermaDBConfig.Descriptions.PERMADB_ENABLED) + protected void setPermaDbEnabled(boolean enabled) { + this.permaDbEnabled = enabled; } @Override @@ -455,7 +455,7 @@ public boolean permaIsRevalidate() { } @JsonProperty - @Parameter(names = {"--perma-revalidate"}, description = PermaDBConfig.Descriptions.PERMAREVALIDATE, arity = 1) + @Parameter(names = {"--perma-revalidate"}, description = PermaDBConfig.Descriptions.PERMAREVALIDATE) protected void setPermaRevalidate(boolean permaRevalidate) { this.permaRevalidate = permaRevalidate; } @@ -466,7 +466,7 @@ public boolean permaIsRescanDb() { } @JsonProperty - @Parameter(names = {"--perma-rescan"}, description = PermaDBConfig.Descriptions.PERMARESCAN_DB, arity = 1) + @Parameter(names = {"--perma-rescan"}, description = PermaDBConfig.Descriptions.PERMARESCAN_DB) protected void setPermaRescanDb(boolean permaRescanDb) { this.permaRescanDb = permaRescanDb; } diff --git a/src/main/java/com/iota/iri/model/StateDiff.java b/src/main/java/com/iota/iri/model/StateDiff.java index 0c8ab5f89d..2efc8840ce 100644 --- a/src/main/java/com/iota/iri/model/StateDiff.java +++ b/src/main/java/com/iota/iri/model/StateDiff.java @@ -58,4 +58,14 @@ public void readMetadata(byte[] bytes) { public boolean merge() { return false; } + @Override + public Persistable mergeTwo(Persistable nrTwo){ + return null; + } + + @Override + public boolean isEmpty() { + return state == null || state.isEmpty(); + } + } diff --git a/src/main/java/com/iota/iri/model/persistables/CountedTransaction.java b/src/main/java/com/iota/iri/model/persistables/CountedTransaction.java deleted file mode 100644 index 51cec8af50..0000000000 --- a/src/main/java/com/iota/iri/model/persistables/CountedTransaction.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.iota.iri.model.persistables; - -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.Hash; -import com.iota.iri.model.HashFactory; -import com.iota.iri.utils.Serializer; - -import java.nio.ByteBuffer; -import java.util.Arrays; - -public class CountedTransaction extends Transaction { - - public static int counterSize = Long.BYTES; - public long storageReferenceCounter = 0l; - - public static CountedTransaction fromTransaction(Transaction tx, long storageReferenceCounter){ - CountedTransaction toReturn = new CountedTransaction(); - toReturn.bytes = tx.bytes(); - byte[] metadataBuffer = tx.metadata(); - ByteBuffer bb = ByteBuffer.allocate(metadataBuffer.length + counterSize); - bb.putLong(storageReferenceCounter); - bb.put(metadataBuffer); - toReturn.readMetadata(bb.array()); - - return toReturn; - } - - - @Override - public byte[] metadata() { - byte[] metadataBuffer = super.metadata(); - ByteBuffer bb = ByteBuffer.allocate(metadataBuffer.length + counterSize); - bb.putLong(storageReferenceCounter); - bb.put(metadataBuffer); - return bb.array(); - } - - - @Override - public void readMetadata(byte[] bytes) { - if(bytes != null) { - byte[] counterBytes = new byte[counterSize]; - System.arraycopy(bytes, 0, counterBytes, 0, counterSize); - storageReferenceCounter = Serializer.getLong(counterBytes, 0); - byte[] metadataBytes = new byte[bytes.length - counterSize]; - System.arraycopy(bytes, counterSize, metadataBytes, 0, metadataBytes.length); - - - super.readMetadata(metadataBytes); - } - - - } -} diff --git a/src/main/java/com/iota/iri/model/persistables/Hashes.java b/src/main/java/com/iota/iri/model/persistables/Hashes.java index 408c504648..3c5e6047e3 100644 --- a/src/main/java/com/iota/iri/model/persistables/Hashes.java +++ b/src/main/java/com/iota/iri/model/persistables/Hashes.java @@ -66,4 +66,21 @@ public void readMetadata(byte[] bytes) { public boolean merge() { return true; } + + @Override + public Persistable mergeTwo(Persistable nrTwo){ + if(nrTwo instanceof Hashes){ + Set setTwo = ((Hashes) nrTwo).set; + set.addAll(setTwo); + return this; + } + + return null; + } + + @Override + public boolean isEmpty() { + return set.isEmpty(); + } + } diff --git a/src/main/java/com/iota/iri/model/persistables/Milestone.java b/src/main/java/com/iota/iri/model/persistables/Milestone.java index ed96bc29fb..3938199c52 100644 --- a/src/main/java/com/iota/iri/model/persistables/Milestone.java +++ b/src/main/java/com/iota/iri/model/persistables/Milestone.java @@ -58,4 +58,15 @@ public void readMetadata(byte[] bytes) { public boolean merge() { return false; } + @Override + public Persistable mergeTwo(Persistable nrTwo){ + return null; + } + + @Override + public boolean isEmpty() { + return hash == null || index == null; + } + + } diff --git a/src/main/java/com/iota/iri/model/persistables/SpentAddress.java b/src/main/java/com/iota/iri/model/persistables/SpentAddress.java index 3c2372d474..57fa3aed67 100644 --- a/src/main/java/com/iota/iri/model/persistables/SpentAddress.java +++ b/src/main/java/com/iota/iri/model/persistables/SpentAddress.java @@ -32,4 +32,15 @@ public boolean merge() { public boolean exists() { return exists; } + + @Override + public Persistable mergeTwo(Persistable nrTwo){ + return null; + } + + @Override + public boolean isEmpty() { + return exists; + } + } diff --git a/src/main/java/com/iota/iri/model/persistables/Transaction.java b/src/main/java/com/iota/iri/model/persistables/Transaction.java index 5b81fb70b0..93e1ea5894 100644 --- a/src/main/java/com/iota/iri/model/persistables/Transaction.java +++ b/src/main/java/com/iota/iri/model/persistables/Transaction.java @@ -219,4 +219,14 @@ public void readMetadata(byte[] bytes) { public boolean merge() { return false; } + + @Override + public Persistable mergeTwo(Persistable nrTwo){ + return null; + } + @Override + public boolean isEmpty() { + return bytes == null || bytes.length == 0; + } + } diff --git a/src/main/java/com/iota/iri/storage/CountedPermanentPersitenceProvider.java b/src/main/java/com/iota/iri/storage/CountedPermanentPersitenceProvider.java new file mode 100644 index 0000000000..35c32c87c1 --- /dev/null +++ b/src/main/java/com/iota/iri/storage/CountedPermanentPersitenceProvider.java @@ -0,0 +1,26 @@ +package com.iota.iri.storage; + +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.persistables.Hashes; + +public interface CountedPermanentPersitenceProvider { + + + void init() throws Exception; + boolean isAvailable(); + void shutdown(); + boolean saveTransaction(TransactionViewModel model, Indexable index) throws Exception; + + boolean incrementTransactions(Indexable[] indexes) throws Exception; + + boolean decrementTransactions(Indexable[] indexes) throws Exception; + + boolean setCounter(Indexable index, long counter) throws Exception; + + TransactionViewModel getTransaction(Indexable index) throws Exception; + + Hashes findAddress(Indexable indexes) throws Exception; + Hashes findBundle(Indexable indexes) throws Exception; + Hashes findTag(Indexable indexes) throws Exception; + Hashes findApprovee(Indexable indexes) throws Exception; +} diff --git a/src/main/java/com/iota/iri/storage/Persistable.java b/src/main/java/com/iota/iri/storage/Persistable.java index fc5821e504..2a3345bcf5 100644 --- a/src/main/java/com/iota/iri/storage/Persistable.java +++ b/src/main/java/com/iota/iri/storage/Persistable.java @@ -8,4 +8,9 @@ public interface Persistable extends Serializable { byte[] metadata(); void readMetadata(byte[] bytes); boolean merge(); + Persistable mergeTwo(Persistable nrTwo); + //A required add because the persistence providers always return non null values and bytes() fails + //on certain instances. So in the persistence provider there is no way to check if the persistence provider + //actually found the object or not. This is needed if we have multiple persistence providers. + boolean isEmpty(); } diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index 3b1679031f..7e2e6fe08e 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -22,7 +22,6 @@ public class Tangle { public static final Map> COLUMN_FAMILIES = new LinkedHashMap>() {{ put("transaction", Transaction.class); - put("counted-transaction", CountedTransaction.class); put("milestone", Milestone.class); put("stateDiff", StateDiff.class); put("address", Address.class); @@ -35,21 +34,6 @@ public class Tangle { public static final Map.Entry> METADATA_COLUMN_FAMILY = new AbstractMap.SimpleImmutableEntry<>("transaction-metadata", Transaction.class); - public static final Map.Entry> COUNTED_METADATA_COLUMN_FAMILY = - new AbstractMap.SimpleImmutableEntry<>("counted-transaction-metadata", CountedTransaction.class); - -// = -// new LinkedHashMap>() {{ -// put("transaction", Transaction.class); -// put("counted-transaction-metadata", CountedTransaction.class); -// put("milestone", Milestone.class); -// put("stateDiff", StateDiff.class); -// put("address", Address.class); -// put("approvee", Approvee.class); -// put("bundle", Bundle.class); -// put("obsoleteTag", ObsoleteTag.class); -// put("tag", Tag.class); -// }}; private final List persistenceProviders = new ArrayList<>(); private final List messageQueueProviders = new ArrayList<>(); @@ -82,13 +66,33 @@ public void shutdown() throws Exception { } public Persistable load(Class model, Indexable index) throws Exception { - Persistable out = null; + LinkedList outlist = new LinkedList<>(); for(PersistenceProvider provider: this.persistenceProviders) { - if((out = provider.get(model, index)) != null) { - break; + Persistable result = provider.get(model, index); + + if (result != null && !result.isEmpty()) { + if(result.merge()){ + + outlist.add(result); + }else{ + //If it is a non-mergeable result then there is no need to ask another provider again. + // Return immediately + return result; + } } } - return out; + Persistable p = outlist.stream().reduce(null, (a,b) -> { + if(a == null){ + return b; + } + return a.mergeTwo(b); + }); + if(p==null){ + p = (Persistable) model.newInstance(); + } + + return p; + } public Boolean saveBatch(List> models) throws Exception { diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java new file mode 100644 index 0000000000..e522d689dd --- /dev/null +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -0,0 +1,555 @@ +package com.iota.iri.storage.rocksDB; + +import com.google.common.annotations.VisibleForTesting; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; +import com.iota.iri.model.HashFactory; +import com.iota.iri.model.persistables.*; +import com.iota.iri.model.persistables.Transaction; +import com.iota.iri.storage.CountedPermanentPersitenceProvider; +import com.iota.iri.storage.Indexable; +import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.PersistenceProvider; +import com.iota.iri.utils.IotaIOUtils; +import com.iota.iri.utils.Pair; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.SystemUtils; +import org.rocksdb.*; +import org.rocksdb.util.SizeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.file.Paths; +import java.security.SecureRandom; +import java.util.*; + +public class RocksDBPPPImpl implements CountedPermanentPersitenceProvider, PersistenceProvider { + private static final Logger log = LoggerFactory.getLogger(RocksDBPPPImpl.class); + private static final int BLOOM_FILTER_BITS_PER_KEY = 10; + /**A delimeter for separating hashes within a byte stream*/ + private static final byte delimiter = ",".getBytes()[0]; + private final List columnFamilyHandles = new ArrayList<>(); + + private final String dbPath; + private final String logPath; + private final int cacheSize; + + public static String COUNTED_TRANSACTION = "counted-transaction"; + public static String COUNTER_INDEX = "counter-index"; + public static String ADDRESS_INDEX = "address-index"; + public static String BUNDLE_INDEX = "bundle-index"; + + public static String TAG_INDEX = "tag-index"; + public static String APPROVEE_INDEX = "approvee-index"; + + @VisibleForTesting + public Map columnMap = new HashMap<>(); + + private RocksDB db; + // DBOptions is only used in initDB(). However, it is closeable - so we keep a reference for shutdown. + private DBOptions options; + private BloomFilter bloomFilter; + private boolean available; + + public RocksDBPPPImpl(String dbPath, String logPath, int cacheSize) { + this.dbPath = dbPath; + this.logPath = logPath; + this.cacheSize = cacheSize; + + } + + @Override + public void init() throws Exception { + log.info("Initializing Database on " + dbPath); + initDB(dbPath, logPath); + available = true; + log.info("RocksDB permanent persistence provider initialized."); + } + + @Override + public boolean isAvailable() { + return this.available; + } + + @Override + public void shutdown() { + for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { + IotaIOUtils.closeQuietly(columnFamilyHandle); + } + IotaIOUtils.closeQuietly(db, options, bloomFilter); + } + + + //----------- PersistenceProvider only overrides --------------- + @Override + public boolean save(Persistable model, Indexable index) throws Exception { + return false; + } + + @Override + public void delete(Class model, Indexable index) throws Exception { + //Do Nothing + } + + @Override + public boolean update(Persistable model, Indexable index, String item) throws Exception { + return false; + } + + @Override + public boolean exists(Class model, Indexable key) throws Exception { + return false; + } + + @Override + public Pair latest(Class model, Class indexModel) throws Exception { + return null; + } + + @Override + public Set keysWithMissingReferences(Class modelClass, Class otherClass) throws Exception { + return null; + } + + @Override + public Persistable get(Class model, Indexable index) throws Exception { + Persistable object = (Persistable) model.newInstance(); + + if(object instanceof Transaction) { + TransactionViewModel tvm = getTransaction(index); + if(tvm == null){ + return object; + } + //These 'manually' fill the metadata from the transaction data. + tvm.getAddressHash(); + tvm.getBundleHash(); + tvm.getBranchTransactionHash(); + tvm.getTrunkTransactionHash(); + tvm.getObsoleteTagValue(); + tvm.getObsoleteTagValue(); + tvm.getTagValue(); + tvm.setAttachmentData(); + tvm.setMetadata(); + + return tvm.getTransaction(); + } + + if(object instanceof Bundle){ + Bundle toReturn = new Bundle(); + toReturn.set = findBundle(index).set; + return toReturn; + } + if(object instanceof Address){ + Address toReturn = new Address(); + toReturn.set = findAddress(index).set; + return toReturn; + } + if(object instanceof Tag){ + Tag toReturn = new Tag(); + toReturn.set = findTag(index).set; + return toReturn; + } + if(object instanceof Approvee){ + Approvee toReturn = new Approvee(); + toReturn.set = findApprovee(index).set; + return toReturn; + } + return object; + } + + public void loopAddressIndex(){ + try (RocksIterator iterator = db.newIterator(columnMap.get(ADDRESS_INDEX))) { + + + + + iterator.seekToFirst(); + while(iterator.isValid()) { + Hashes h = new Hashes(); + byte[] value = iterator.value(); + h.read(value); + Hash i = HashFactory.ADDRESS.create(iterator.value()); + log.info(i.toString() + " " + h.toString()); + iterator.next(); + } + + } + } + + @Override + public boolean mayExist(Class model, Indexable index) throws Exception { + return false; + } + + @Override + public long count(Class model) throws Exception { + return 0; + } + + @Override + public Set keysStartingWith(Class modelClass, byte[] value) { + return null; + } + + @Override + public Persistable seek(Class model, byte[] key) throws Exception { + return null; + } + + @Override + public Pair next(Class model, Indexable index) throws Exception { + return null; + } + + @Override + public Pair previous(Class model, Indexable index) throws Exception { + return null; + } + + @Override + public Pair first(Class model, Class indexModel) throws Exception { + return null; + } + + @Override + public boolean saveBatch(List> models) throws Exception { + return false; + } + + @Override + public void deleteBatch(Collection>> models) throws Exception { + + } + + @Override + public void clear(Class column) throws Exception { + + } + + @Override + public void clearMetadata(Class column) throws Exception { + + } + + @Override + public List loadAllKeysFromTable(Class model) { + return null; + } + //END----------- PersistenceProvider only overrides --------------- + + + @Override + public boolean saveTransaction(TransactionViewModel model, Indexable index) throws Exception { + long counter = getCounter(index); + //check if the transaction does not exists + if(counter <= 0){ + try (WriteBatch writeBatch = new WriteBatch(); + WriteOptions writeOptions = new WriteOptions()) { + byte[] txBytes = model.getBytes(); + + writeBatch.put(columnMap.get(COUNTED_TRANSACTION), index.bytes(), txBytes); + + //Prepare counted index + ByteBuffer counterBuffer = ByteBuffer.allocate(Long.BYTES); + counterBuffer.putLong(1L); + writeBatch.put(columnMap.get(COUNTER_INDEX),index.bytes(), counterBuffer.array()); + + //addToIndex(writeBatch, columnMap.get(COUNTER_INDEX), counterBuffer.array() , index); + addToIndex(writeBatch, columnMap.get(ADDRESS_INDEX), model.getAddressHash(), index); + addToIndex(writeBatch, columnMap.get(TAG_INDEX), model.getTagValue(), index); + addToIndex(writeBatch, columnMap.get(BUNDLE_INDEX), model.getBundleHash(), index); + + //TODO only add when relevant in reference. + addToIndex(writeBatch, columnMap.get(APPROVEE_INDEX), model.getTrunkTransactionHash(), index); + addToIndex(writeBatch, columnMap.get(APPROVEE_INDEX), model.getBranchTransactionHash(), index); + db.write(writeOptions, writeBatch); + } + + + } + return false; + } + + /** + * This is a dump method and does not take into account the mechanisms for storing + * @param indexes + * @return + * @throws Exception + */ + + @Override + public boolean incrementTransactions(Indexable[] indexes) throws Exception { + try (WriteBatch writeBatch = new WriteBatch(); + WriteOptions writeOptions = new WriteOptions()) { + for (Indexable i: indexes) { + //sorry no merge for long, the C++ lib has a merge function for Uin64 but that is not (natively)supported by Java. + long increamentedCounter = getCounter(i) + 1; + writeBatch.put(columnMap.get(COUNTER_INDEX), i.bytes(), ByteBuffer.allocate(Long.BYTES).putLong(increamentedCounter).array() ); + } + + db.write(writeOptions, writeBatch); + + } + return false; + } + + @VisibleForTesting + public void safeDeleteTransaction(WriteBatch writeBatch, Indexable key) throws Exception { + byte[] keyBytes = key.bytes(); + TransactionViewModel tx = getTransaction(key); + writeBatch.delete(columnMap.get(COUNTER_INDEX), keyBytes); + writeBatch.delete(columnMap.get(COUNTED_TRANSACTION), keyBytes); + + removeFromIndex(writeBatch, columnMap.get(ADDRESS_INDEX), tx.getAddressHash(), key); + removeFromIndex(writeBatch, columnMap.get(TAG_INDEX), tx.getTagValue(), key); + removeFromIndex(writeBatch, columnMap.get(BUNDLE_INDEX), tx.getBundleHash(), key); + removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getTrunkTransactionHash(), key); + removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getBranchTransactionHash(), key); + + } + @VisibleForTesting + public void removeFromIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) throws Exception{ + byte[] indexBytes = indexValue.bytes(); + byte[] result = db.get(column, key.bytes()); + if(result != null) { + if(result.length <= indexBytes.length){ + //when it is the last one to delete just delete the entire index key space. + writeBatch.delete(column, key.bytes()); + }else{ + int keyLoc = indexOf(result, indexBytes); + if(keyLoc >= 0){ //Does exists + ByteBuffer buffer = ByteBuffer.allocate( result.length - indexBytes.length - 1); + byte[] subarray1 = ArrayUtils.subarray(result, 0, keyLoc - 1); + buffer.put(subarray1); + byte[] subarray2 = ArrayUtils.subarray(result, keyLoc + indexBytes.length + 1, result.length); + if(subarray1.length > 0 && subarray2.length > 0){ + buffer.put(delimiter); + buffer.put(subarray2); + } + + + writeBatch.put(column, key.bytes(), buffer.array()); + } + + } + + + } + } + @VisibleForTesting + public void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) throws Exception{ + byte[] indexBytes = indexValue.bytes(); + byte[] dbResult = db.get(column, key.bytes()); + if(dbResult == null) { + dbResult = new byte[0]; + } + + int keyLoc = indexOf(dbResult, indexBytes); + if(keyLoc == -1){//not found + + ByteBuffer buffer = ByteBuffer.allocate(dbResult.length + indexBytes.length + (dbResult.length > 0 ? 1 : 0));//+1 delimiter length + + buffer.put(dbResult); + if(dbResult.length > 0){ + //Means we add on the end of the current stream. + //To be compatible with Hashes.read(byte[]) + buffer.put(delimiter); + } + buffer.put(indexBytes); + + writeBatch.put(column, key.bytes(), buffer.array()); + } + } + + + public int indexOf(byte[] outerArray, byte[] smallerArray) { + for(int i = 0; i < outerArray.length - smallerArray.length+1; ++i) { + boolean found = true; + for(int j = 0; j < smallerArray.length; ++j) { + if (outerArray[i+j] != smallerArray[j]) { + found = false; + break; + } + } + if (found) return i; + } + return -1; + } + + + + @Override + public boolean decrementTransactions(Indexable[] indexes) throws Exception { + try (WriteBatch writeBatch = new WriteBatch(); + WriteOptions writeOptions = new WriteOptions()) { + for (Indexable i: indexes) { + //sorry no merge for long, the C++ lib has a merge function for Uin64 but that is not (natively)supported by Java. + long decrementedCounter = getCounter(i) -1; + if(decrementedCounter <= 0){ + safeDeleteTransaction(writeBatch, i); + }else { + writeBatch.put(columnMap.get(COUNTER_INDEX), i.bytes(), ByteBuffer.allocate(Long.BYTES).putLong(decrementedCounter).array()); + } + } + + db.write(writeOptions, writeBatch); + } + return false; + } + + @Override + public boolean setCounter(Indexable index, long counter) throws Exception { + try (WriteBatch writeBatch = new WriteBatch(); + WriteOptions writeOptions = new WriteOptions()) { + if(counter <= 0){ + safeDeleteTransaction(writeBatch, index); + }else { + writeBatch.put(columnMap.get(COUNTER_INDEX), index.bytes(), ByteBuffer.allocate(Long.BYTES).putLong(counter).array()); + } + db.write(writeOptions, writeBatch); + } + return false; + } + + + public long getCounter(Indexable index) throws Exception { + if(index == null){return -1;} + byte[] dbResult = db.get(columnMap.get(COUNTER_INDEX), index.bytes()); + if(dbResult != null && dbResult.length == Long.BYTES) { + long counter = ByteBuffer.wrap(dbResult).getLong(); + return counter; + } + return 0; + } + + @Override + public TransactionViewModel getTransaction(Indexable index) throws Exception { + if(index == null){return null;} + byte[] dbResult = db.get(columnMap.get(COUNTED_TRANSACTION), index.bytes()); + if(dbResult != null && dbResult.length > 0){ + return new TransactionViewModel(TransactionViewModel.trits(dbResult), Hash.NULL_HASH); + } + return null; + } + + @Override + public Hashes findAddress(Indexable index) throws Exception { + return getIndex(columnMap.get(ADDRESS_INDEX), index); + } + @Override + public Hashes findBundle(Indexable index) throws Exception { + return getIndex(columnMap.get(BUNDLE_INDEX), index); + } + @Override + public Hashes findTag(Indexable index) throws Exception { + return getIndex(columnMap.get(TAG_INDEX), index); + } + @Override + public Hashes findApprovee(Indexable index) throws Exception { + return getIndex(columnMap.get(APPROVEE_INDEX), index); + } + + + private Hashes getIndex(ColumnFamilyHandle column, Indexable index) throws Exception { + if(index == null){return null;} + Hashes found = new Hashes(); + byte[] result = db.get(column, index.bytes()); + if(result != null) { + found.read(result); + + } + return found; + } + + private void initDB(String path, String logPath) throws Exception { + try { + try { + RocksDB.loadLibrary(); + } catch (Exception e) { + if (SystemUtils.IS_OS_WINDOWS) { + log.error("Error loading RocksDB library. Please ensure that " + + "Microsoft Visual C++ 2015 Redistributable Update 3 " + + "is installed and updated"); + } + throw e; + } + + File pathToLogDir = Paths.get(logPath).toFile(); + if (!pathToLogDir.exists() || !pathToLogDir.isDirectory()) { + boolean success = pathToLogDir.mkdir(); + if (!success) { + log.warn("Unable to make directory: {}", pathToLogDir); + } + } + + int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); + RocksEnv.getDefault() + .setBackgroundThreads(numThreads, RocksEnv.FLUSH_POOL) + .setBackgroundThreads(numThreads, RocksEnv.COMPACTION_POOL); + + options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true) + .setDbLogDir(logPath) + .setMaxLogFileSize(SizeUnit.MB) + .setMaxManifestFileSize(SizeUnit.MB) + .setMaxOpenFiles(10000) + .setMaxBackgroundCompactions(1); + + options.setMaxSubcompactions(Runtime.getRuntime().availableProcessors()); + + bloomFilter = new BloomFilter(BLOOM_FILTER_BITS_PER_KEY); + + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig().setFilter(bloomFilter); + blockBasedTableConfig + .setFilter(bloomFilter) + .setCacheNumShardBits(2) + .setBlockSizeDeviation(10) + .setBlockRestartInterval(16) + .setBlockCacheSize(cacheSize * SizeUnit.KB) + .setBlockCacheCompressedNumShardBits(10) + .setBlockCacheCompressedSize(32 * SizeUnit.KB); + + options.setAllowConcurrentMemtableWrite(true); + + MergeOperator mergeOperator = new StringAppendOperator(); + ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions() + .setMergeOperator(mergeOperator) + .setTableFormatConfig(blockBasedTableConfig) + .setMaxWriteBufferNumber(2) + .setWriteBufferSize(2 * SizeUnit.MB); + + List columnFamilyDescriptors = new ArrayList<>(); + //Add default column family. Main motivation is to not change legacy code + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions)); + + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(COUNTED_TRANSACTION.getBytes("UTF-8"), columnFamilyOptions)); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(COUNTER_INDEX.getBytes("UTF-8"), columnFamilyOptions)); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(BUNDLE_INDEX.getBytes("UTF-8"), columnFamilyOptions)); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(TAG_INDEX.getBytes("UTF-8"), columnFamilyOptions)); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(ADDRESS_INDEX.getBytes("UTF-8"), columnFamilyOptions)); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(APPROVEE_INDEX.getBytes("UTF-8"), columnFamilyOptions)); + + + + + db = RocksDB.open(options, path, columnFamilyDescriptors, columnFamilyHandles); + db.enableFileDeletions(true); + + for (ColumnFamilyHandle columnHandler: columnFamilyHandles) { + columnMap.put(new String(columnHandler.getName(), "UTF-8" ), columnHandler); + } + + + } catch (Exception e) { + IotaIOUtils.closeQuietly(db); + throw e; + } + } + + + +} diff --git a/src/test/java/com/iota/iri/model/persistables/CountedTransactionTest.java b/src/test/java/com/iota/iri/model/persistables/CountedTransactionTest.java deleted file mode 100644 index 4f87544677..0000000000 --- a/src/test/java/com/iota/iri/model/persistables/CountedTransactionTest.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.iota.iri.model.persistables; - -import com.iota.iri.TransactionTestUtils; -import com.iota.iri.controllers.TransactionViewModel; -import com.iota.iri.model.Hash; -import com.iota.iri.utils.Converter; -import org.junit.Test; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -public class CountedTransactionTest { - - - - @Test - public void testBytes() { - Transaction t = TransactionTestUtils.getTransaction(); - - Transaction newtx = new Transaction(); - newtx.read(t.bytes()); - newtx.readMetadata(t.metadata()); - CountedTransaction ctx1 = CountedTransaction.fromTransaction(newtx, 5); - - CountedTransaction ctx2 = new CountedTransaction(); - ctx2.read(ctx1.bytes()); - ctx2.readMetadata(ctx1.metadata()); - - - assertArrayEquals("metadata should be the same in the copy", ctx1.metadata(), ctx2.metadata()); - assertArrayEquals("bytes should be the same in the copy", newtx.bytes(), ctx2.bytes()); - } - -// @Test -// public void fromTrits() { -// byte[] trits = TransactionTestUtils.getTransactionTrits(); -// byte[] bytes = Converter.allocateBytesForTrits(trits.length); -// Converter.bytes(trits, bytes); -// -// TransactionViewModel tvm = new TransactionViewModel(trits, Hash.NULL_HASH); -// tvm.getAddressHash(); -// tvm.getTrunkTransactionHash(); -// tvm.getBranchTransactionHash(); -// tvm.getBundleHash(); -// tvm.getTagValue(); -// tvm.getObsoleteTagValue(); -// tvm.setAttachmentData(); -// tvm.setMetadata(); -// -// assertArrayEquals("bytes in the TVM should be unmodified", tvm.getBytes(), bytes); -// -// Transaction tvmTransaction = tvm.getTransaction(); -// -// assertEquals("branch in transaction should be the same as in the tvm", tvmTransaction.branch, tvm.getTransaction().branch); -// } - -} diff --git a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java index 112b4649c6..b1935c5d10 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java @@ -1,76 +1,93 @@ package com.iota.iri.storage.rocksDB; -import com.iota.iri.model.IntegerIndex; -import com.iota.iri.model.persistables.Transaction; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; + import com.iota.iri.storage.Indexable; -import com.iota.iri.storage.Persistable; -import com.iota.iri.storage.Tangle; -import com.iota.iri.utils.Pair; -import org.apache.commons.io.FileUtils; import org.junit.*; -import java.io.File; -import java.util.Arrays; +import java.util.LinkedList; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; + +import static com.iota.iri.TransactionTestUtils.getTransactionHash; +import static com.iota.iri.TransactionTestUtils.getTransactionTrits; +import static com.iota.iri.TransactionTestUtils.getTransactionTritsWithTrunkAndBranch; public class PermaRocksDBPersentenceProviderTest { - private static RocksDBPersistenceProvider rocksDBPersistenceProvider; - private static String dbPath = "tmpdb", dbLogPath = "tmplogs"; + private static RocksDBPPPImpl rocksDBPersistenceProvider; + //private static String dbPath = "tmpdb", dbLogPath = "tmplogs"; + private static String dbPath = "mainnetpermadb", dbLogPath = "mainnetpermadb.log"; @BeforeClass public static void setUpDb() throws Exception { - rocksDBPersistenceProvider = new RocksDBPersistenceProvider( - dbPath, dbLogPath,1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); + rocksDBPersistenceProvider = new RocksDBPPPImpl( + dbPath, dbLogPath,1000); rocksDBPersistenceProvider.init(); } @AfterClass public static void destroyDb() { rocksDBPersistenceProvider.shutdown(); - FileUtils.deleteQuietly(new File(dbPath)); - FileUtils.deleteQuietly(new File(dbLogPath)); - rocksDBPersistenceProvider = null; + //FileUtils.deleteQuietly(new File(dbPath)); + //FileUtils.deleteQuietly(new File(dbLogPath)); + //rocksDBPersistenceProvider = null; } @Before public void setUp() throws Exception { - rocksDBPersistenceProvider.clear(Transaction.class); + } @Test - public void testDeleteBatch() throws Exception { - Persistable tx = new Transaction(); - byte[] bytes = new byte[Transaction.SIZE]; - Arrays.fill(bytes, (byte) 1); - tx.read(bytes); - tx.readMetadata(bytes); - List> models = IntStream.range(1, 1000) - .mapToObj(i -> new Pair<>((Indexable) new IntegerIndex(i), tx)) - .collect(Collectors.toList()); - - rocksDBPersistenceProvider.saveBatch(models); - - List>> modelsToDelete = models.stream() - .filter(entry -> ((IntegerIndex) entry.low).getValue() < 900) - .map(entry -> new Pair<>(entry.low, entry.hi.getClass())) - .collect(Collectors.toList()); - - rocksDBPersistenceProvider.deleteBatch(modelsToDelete); - - for (Pair> model : modelsToDelete) { - Assert.assertNull("value at index " + ((IntegerIndex) model.low).getValue() + " should be deleted", - rocksDBPersistenceProvider.get(model.hi, model.low).bytes()); - } + public void readData() throws Exception { + rocksDBPersistenceProvider.loopAddressIndex(); + - List indexes = IntStream.range(900, 1000) - .mapToObj(i -> new IntegerIndex(i)) - .collect(Collectors.toList()); + } + + + @Test + public void testSelectiveTransactionModel() throws Exception { - for (IntegerIndex index : indexes) { - Assert.assertArrayEquals("saved bytes are not as expected in index " + index.getValue(), tx.bytes(), - rocksDBPersistenceProvider.get(Transaction.class, index).bytes()); + TransactionViewModel transaction, transaction1, transaction2, transaction3, transaction4; + transaction = new TransactionViewModel(getTransactionTrits(), Hash.NULL_HASH); + transaction1 = new TransactionViewModel(getTransactionTritsWithTrunkAndBranch(transaction.getHash(), + transaction.getHash()), getTransactionHash()); + transaction2 = new TransactionViewModel(getTransactionTritsWithTrunkAndBranch(transaction1.getHash(), + transaction1.getHash()), getTransactionHash()); + transaction3 = new TransactionViewModel(getTransactionTritsWithTrunkAndBranch(transaction2.getHash(), + transaction1.getHash()), getTransactionHash()); + transaction4 = new TransactionViewModel(getTransactionTritsWithTrunkAndBranch(transaction2.getHash(), + transaction3.getHash()), getTransactionHash()); + + List mm = new LinkedList<>(); + mm.add(transaction); + mm.add(transaction1); + mm.add(transaction2); + mm.add(transaction3); + mm.add(transaction4); + + for(TransactionViewModel v: mm){ + System.out.println(v.getHash() + " \t " + v.getAddressHash()); + rocksDBPersistenceProvider.saveTransaction(v, v.getHash()); } + + + + rocksDBPersistenceProvider.incrementTransactions(new Indexable[]{transaction.getHash()}); + long count = rocksDBPersistenceProvider.getCounter(transaction.getHash()); + Assert.assertEquals(count, 2); + + //See if all indexes are updated. + Assert.assertTrue(rocksDBPersistenceProvider.findAddress(transaction2.getAddressHash()).set.contains(transaction2.getHash())); + Assert.assertTrue(rocksDBPersistenceProvider.findTag(transaction2.getTagValue()).set.contains(transaction2.getHash())); + Assert.assertTrue(rocksDBPersistenceProvider.findBundle(transaction2.getBundleHash()).set.contains(transaction2.getHash())); + Assert.assertTrue(rocksDBPersistenceProvider.findApprovee(transaction2.getTrunkTransactionHash()).set.contains(transaction2.getHash())); + Assert.assertTrue(rocksDBPersistenceProvider.findApprovee(transaction2.getBranchTransactionHash()).set.contains(transaction2.getHash())); + + //Test decremeant delete; + rocksDBPersistenceProvider.decrementTransactions(new Indexable[]{transaction3.getHash()}); + Assert.assertNull(rocksDBPersistenceProvider.getTransaction(transaction3.getHash())); + } } diff --git a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java index ea7cdccc96..5338fad97b 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java @@ -1,7 +1,6 @@ package com.iota.iri.storage.rocksDB; import com.iota.iri.model.IntegerIndex; -import com.iota.iri.model.persistables.CountedTransaction; import com.iota.iri.model.persistables.Transaction; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; @@ -26,7 +25,7 @@ public class RocksDBPersistenceProviderTest { @BeforeClass public static void setUpDb() throws Exception { rocksDBPersistenceProvider = new RocksDBPersistenceProvider( - dbPath, dbLogPath,1000, Tangle.COLUMN_FAMILIES, Tangle.COUNTED_METADATA_COLUMN_FAMILY); + dbPath, dbLogPath,1000, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); rocksDBPersistenceProvider.init(); } @@ -53,7 +52,7 @@ public void testDeleteBatch() throws Exception { List> models = IntStream.range(1, 1000) - .mapToObj(i -> new Pair<>((Indexable) new IntegerIndex(i), (Persistable)CountedTransaction.fromTransaction(tx,i))) + .mapToObj(i -> new Pair<>((Indexable) new IntegerIndex(i), (Persistable)tx)) .collect(Collectors.toList()); rocksDBPersistenceProvider.saveBatch(models); @@ -76,7 +75,7 @@ public void testDeleteBatch() throws Exception { for (IntegerIndex index : indexes) { Assert.assertArrayEquals("saved bytes are not as expected in index " + index.getValue(), tx.bytes(), - rocksDBPersistenceProvider.get(CountedTransaction.class, index).bytes()); + rocksDBPersistenceProvider.get(Transaction.class, index).bytes()); } } } \ No newline at end of file From d69046319ed2656a2ef7f1f4bf03eaf2598ca517 Mon Sep 17 00:00:00 2001 From: olaf Date: Wed, 21 Aug 2019 21:28:39 +0200 Subject: [PATCH 03/26] - Added a separate permanent storage provider for permanent storage - Added functions to the tangle to handle permanent storage - Added API's: * pinTransactionHashes (hashes:[]) * pinTransactionTrytes (trytes:[]) * increaseTransactionCount (hashes:[]) * decreaseTransactionCount (hashes:[]) * getTransactionsCount (hashes:[]) - or others to test this branch: set default settings for permanent storage --- src/main/java/com/iota/iri/Iota.java | 8 +- .../com/iota/iri/conf/BaseIotaConfig.java | 2 +- src/main/java/com/iota/iri/service/API.java | 123 ++++++++++++++++++ .../java/com/iota/iri/service/ApiCommand.java | 23 +++- .../iri/service/dto/LongValuesResponse.java | 20 +++ ...java => PermanentPersistenceProvider.java} | 8 +- .../java/com/iota/iri/storage/Tangle.java | 49 +++++++ .../iri/storage/rocksDB/RocksDBPPPImpl.java | 18 +-- .../PermaRocksDBPersentenceProviderTest.java | 4 +- 9 files changed, 232 insertions(+), 23 deletions(-) create mode 100644 src/main/java/com/iota/iri/service/dto/LongValuesResponse.java rename src/main/java/com/iota/iri/storage/{CountedPermanentPersitenceProvider.java => PermanentPersistenceProvider.java} (74%) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 7a30ebb431..55f7d52767 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -328,11 +328,13 @@ private void initializeTangle() { if(configuration.isSelectivePermaEnabled()){ switch (configuration.getPermaMainDb()) { case "rocksdb": { - tangle.addPersistenceProvider(new RocksDBPPPImpl( + RocksDBPPPImpl ppp = new RocksDBPPPImpl( configuration.getPermaDbPath(), configuration.getPermaDbLogPath(), - configuration.getPermaDbCacheSize()) - ); + configuration.getPermaDbCacheSize()); + + tangle.addPersistenceProvider(ppp); + tangle.addPermanentPersistenceProvider(ppp); break; } default: { diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index 832e1c612c..f14f9856c2 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -948,7 +948,7 @@ public interface Defaults { boolean RESCAN_DB = false; //PERMADB - boolean PERMADB_ENABLED = false; + boolean PERMADB_ENABLED = true; String PERMADB_PATH = "mainnetpermadb"; String PERMADB_LOG_PATH = "mainnetpermadb.log"; int PERMADB_CACHE_SIZE = 100_000; diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 3194828f91..723f733f2a 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -196,6 +196,14 @@ public API(IotaConfig configuration, IXI ixi, TransactionRequester transactionRe commandRoute.put(ApiCommand.GET_MISSING_TRANSACTIONS, getMissingTransactions()); commandRoute.put(ApiCommand.CHECK_CONSISTENCY, checkConsistency()); commandRoute.put(ApiCommand.WERE_ADDRESSES_SPENT_FROM, wereAddressesSpentFrom()); + + //----- permanent storage------ + commandRoute.put(ApiCommand.PIN_TRANSACTION_HASHES, pinTransactionHashes()); + commandRoute.put(ApiCommand.PINNED_TRANSACTIONS_COUNT, pinnedTransactionsCount()); + commandRoute.put(ApiCommand.INCREASE_TRANSACTIONS_COUNT, increaseTransactionsCount()); + commandRoute.put(ApiCommand.DECREASE_TRANSACTIONS_COUNT, decreaseTransactionsCount()); + commandRoute.put(ApiCommand.PIN_TRANSACTIONS_TRYTES, pinTransactionTrytes()); + } /** @@ -315,6 +323,62 @@ private AbstractResponse wereAddressesSpentFromStatement(List addresses) return WereAddressesSpentFrom.create(states); } + // ----------- permanent storage -------- + + @Document(name="pinTransactionTrytes") + public AbstractResponse pinTransactionTrytesStatement(List trytes) throws Exception { + final List elements = convertTrytes(trytes); + for (final TransactionViewModel transactionViewModel : elements) { + //store transactions + tangle.pinTransaction(transactionViewModel); + } + return AbstractResponse.createEmptyResponse(); + } + + @Document(name="pinTransactionHashes") + private AbstractResponse pinTransactionHashStatement(List transactionsList) throws Exception { + final List transactions = transactionsList.stream().map(HashFactory.TRANSACTION::create).collect(Collectors.toList()); + + for (Hash transaction : transactions) { + tangle.pinTransaction(transaction); + } + + return AbstractResponse.createEmptyResponse(); + } + + @Document(name="pinnedTransactionsCount") + private AbstractResponse pinnedTransactionsCountStatement(List transactionsList) throws Exception { + long[] toReturn = new long[transactionsList.size()]; + for (int i=0; i< toReturn.length;i++) { + toReturn[i] = tangle.getTransactionStorageCounter(HashFactory.TRANSACTION.create(transactionsList.get(i))); + } + return LongValuesResponse.create(toReturn); + } + + @Document(name="increaseTransactionsCount") + private AbstractResponse increaseTransactionsCountStatement(List transactionsList) throws Exception { + Hash[] toReturn = new Hash[transactionsList.size()]; + for (int i=0; i< toReturn.length;i++) { + toReturn[i] = HashFactory.TRANSACTION.create(transactionsList.get(i)); + } + tangle.incrementTransactions(toReturn); + return AbstractResponse.createEmptyResponse(); + } + + @Document(name="decreaseTransactionsCount") + private AbstractResponse decreaseTransactionsCountStatement(List transactionsList) throws Exception { + Hash[] toReturn = new Hash[transactionsList.size()]; + for (int i=0; i< toReturn.length;i++) { + toReturn[i] = HashFactory.TRANSACTION.create(transactionsList.get(i)); + } + tangle.decrementTransactions(toReturn); + return AbstractResponse.createEmptyResponse(); + } + + // -------------------------------------- + + + /** * Walks back from the hash until a tail transaction has been found or transaction aprovee is not found. * A tail transaction is the first transaction in a bundle, thus with index = 0 @@ -1730,4 +1794,63 @@ private List convertTrytes(List trytes) { return elements; } + // ------ permanode ---------- + + private Function, AbstractResponse> pinTransactionTrytes() { + return request -> { + final List transactionTrytes = getParameterAsList(request,"trytes", TRYTES_SIZE); + try { + return pinTransactionTrytesStatement(transactionTrytes); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + private Function, AbstractResponse> pinTransactionHashes() { + return request -> { + final List txids = getParameterAsList(request,"hashes", HASH_SIZE); + try { + return pinTransactionHashStatement(txids); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + private Function, AbstractResponse> pinnedTransactionsCount() { + return request -> { + final List txids = getParameterAsList(request,"hashes", HASH_SIZE); + try { + return pinnedTransactionsCountStatement(txids); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + private Function, AbstractResponse> increaseTransactionsCount() { + return request -> { + final List txids = getParameterAsList(request,"hashes", HASH_SIZE); + try { + return increaseTransactionsCountStatement(txids); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + private Function, AbstractResponse> decreaseTransactionsCount() { + return request -> { + final List txids = getParameterAsList(request,"hashes", HASH_SIZE); + try { + return decreaseTransactionsCountStatement(txids); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }; + } + + // --------------------------- + } diff --git a/src/main/java/com/iota/iri/service/ApiCommand.java b/src/main/java/com/iota/iri/service/ApiCommand.java index 280d42be8d..a80822b8ca 100644 --- a/src/main/java/com/iota/iri/service/ApiCommand.java +++ b/src/main/java/com/iota/iri/service/ApiCommand.java @@ -81,7 +81,7 @@ public enum ApiCommand { * Stop attaching to the tangle */ INTERRUPT_ATTACHING_TO_TANGLE("interruptAttachingToTangle"), - + /** * Temporary remove a neighbor from this node */ @@ -91,11 +91,28 @@ public enum ApiCommand { * Store a transaction on this node, without broadcasting */ STORE_TRANSACTIONS("storeTransactions"), - + /** * Check if an address has been spent from */ - WERE_ADDRESSES_SPENT_FROM("wereAddressesSpentFrom"); + WERE_ADDRESSES_SPENT_FROM("wereAddressesSpentFrom"), + + // ---------- permanent storage + + /** + * Pins a transaction + */ + PIN_TRANSACTION_HASHES("pinTransactionHashes"), + + PINNED_TRANSACTIONS_COUNT("pinnedTransactionsCount"), + + INCREASE_TRANSACTIONS_COUNT("increaseTransactionsCount"), + + DECREASE_TRANSACTIONS_COUNT("decreaseTransactionsCount"), + + PIN_TRANSACTIONS_TRYTES("pinTransactionsTrytes"); + + private String name; diff --git a/src/main/java/com/iota/iri/service/dto/LongValuesResponse.java b/src/main/java/com/iota/iri/service/dto/LongValuesResponse.java new file mode 100644 index 0000000000..6f084730c3 --- /dev/null +++ b/src/main/java/com/iota/iri/service/dto/LongValuesResponse.java @@ -0,0 +1,20 @@ +package com.iota.iri.service.dto; + +public class LongValuesResponse extends AbstractResponse { + /** + * States of the specified addresses in Boolean + * Order of booleans is equal to order of the supplied addresses. + */ + private long[] values; + + + public static AbstractResponse create(long[] values) { + LongValuesResponse res = new LongValuesResponse(); + res.values = values; + return res; + } + + public long[] getValue() { + return this.values; + } +} diff --git a/src/main/java/com/iota/iri/storage/CountedPermanentPersitenceProvider.java b/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java similarity index 74% rename from src/main/java/com/iota/iri/storage/CountedPermanentPersitenceProvider.java rename to src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java index 35c32c87c1..6c8fea4e43 100644 --- a/src/main/java/com/iota/iri/storage/CountedPermanentPersitenceProvider.java +++ b/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java @@ -3,7 +3,7 @@ import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.persistables.Hashes; -public interface CountedPermanentPersitenceProvider { +public interface PermanentPersistenceProvider { void init() throws Exception; @@ -11,9 +11,11 @@ public interface CountedPermanentPersitenceProvider { void shutdown(); boolean saveTransaction(TransactionViewModel model, Indexable index) throws Exception; - boolean incrementTransactions(Indexable[] indexes) throws Exception; + void incrementTransactions(Indexable[] indexes) throws Exception; - boolean decrementTransactions(Indexable[] indexes) throws Exception; + void decrementTransactions(Indexable[] indexes) throws Exception; + + long getCounter(Indexable index) throws Exception; boolean setCounter(Indexable index, long counter) throws Exception; diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index 7e2e6fe08e..30bd9a097c 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -1,5 +1,6 @@ package com.iota.iri.storage; +import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.model.StateDiff; import com.iota.iri.model.persistables.*; @@ -36,11 +37,15 @@ public class Tangle { private final List persistenceProviders = new ArrayList<>(); + private final List permanentPersistenceProviders = new ArrayList<>(); private final List messageQueueProviders = new ArrayList<>(); public void addPersistenceProvider(PersistenceProvider provider) { this.persistenceProviders.add(provider); } + public void addPermanentPersistenceProvider(PermanentPersistenceProvider provider) { + this.permanentPersistenceProviders.add(provider); + } /** * Adds {@link com.iota.iri.storage.MessageQueueProvider} that should be notified. @@ -297,4 +302,48 @@ public void clearMetadata(Class column) throws Exception { provider.clearMetadata(column); } } + + // ---------- Permanent storage capabilities -------- + + + public void pinTransaction(TransactionViewModel tvm) throws Exception { + for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { + provider.saveTransaction(tvm, tvm.getHash()); + } + } + + public void pinTransaction(Hash hash) throws Exception { + Transaction tx = (Transaction)load(Transaction.class, hash); + if(!tx.isEmpty()) { + TransactionViewModel tvm = new TransactionViewModel(tx, hash); + pinTransaction(tvm); + } + } + + public void incrementTransactions(Indexable[] indexes) throws Exception { + for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { + provider.incrementTransactions(indexes); + } + } + + public void decrementTransactions(Indexable[] indexes) throws Exception { + for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { + provider.decrementTransactions(indexes); + } + } + + public long getTransactionStorageCounter(Hash index) throws Exception { + long toReturn = 0; + for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { + toReturn = Math.max(0, provider.getCounter(index)); + } + return toReturn; + } + + + + + // ------------------------------------------------- + + } diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index e522d689dd..88c5975e17 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -6,13 +6,12 @@ import com.iota.iri.model.HashFactory; import com.iota.iri.model.persistables.*; import com.iota.iri.model.persistables.Transaction; -import com.iota.iri.storage.CountedPermanentPersitenceProvider; +import com.iota.iri.storage.PermanentPersistenceProvider; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; import com.iota.iri.storage.PersistenceProvider; import com.iota.iri.utils.IotaIOUtils; import com.iota.iri.utils.Pair; -import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.SystemUtils; import org.rocksdb.*; @@ -23,10 +22,9 @@ import java.io.File; import java.nio.ByteBuffer; import java.nio.file.Paths; -import java.security.SecureRandom; import java.util.*; -public class RocksDBPPPImpl implements CountedPermanentPersitenceProvider, PersistenceProvider { +public class RocksDBPPPImpl implements PermanentPersistenceProvider, PersistenceProvider { private static final Logger log = LoggerFactory.getLogger(RocksDBPPPImpl.class); private static final int BLOOM_FILTER_BITS_PER_KEY = 10; /**A delimeter for separating hashes within a byte stream*/ @@ -281,19 +279,18 @@ public boolean saveTransaction(TransactionViewModel model, Indexable index) thro */ @Override - public boolean incrementTransactions(Indexable[] indexes) throws Exception { + public void incrementTransactions(Indexable[] indexes) throws Exception { try (WriteBatch writeBatch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions()) { for (Indexable i: indexes) { //sorry no merge for long, the C++ lib has a merge function for Uin64 but that is not (natively)supported by Java. - long increamentedCounter = getCounter(i) + 1; - writeBatch.put(columnMap.get(COUNTER_INDEX), i.bytes(), ByteBuffer.allocate(Long.BYTES).putLong(increamentedCounter).array() ); + long incrementedCounter = getCounter(i) + 1; + writeBatch.put(columnMap.get(COUNTER_INDEX), i.bytes(), ByteBuffer.allocate(Long.BYTES).putLong(incrementedCounter).array() ); } db.write(writeOptions, writeBatch); } - return false; } @VisibleForTesting @@ -382,7 +379,7 @@ public int indexOf(byte[] outerArray, byte[] smallerArray) { @Override - public boolean decrementTransactions(Indexable[] indexes) throws Exception { + public void decrementTransactions(Indexable[] indexes) throws Exception { try (WriteBatch writeBatch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions()) { for (Indexable i: indexes) { @@ -397,7 +394,6 @@ public boolean decrementTransactions(Indexable[] indexes) throws Exception { db.write(writeOptions, writeBatch); } - return false; } @Override @@ -414,7 +410,7 @@ public boolean setCounter(Indexable index, long counter) throws Exception { return false; } - + @Override public long getCounter(Indexable index) throws Exception { if(index == null){return -1;} byte[] dbResult = db.get(columnMap.get(COUNTER_INDEX), index.bytes()); diff --git a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java index b1935c5d10..a4bef80097 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java @@ -15,8 +15,8 @@ public class PermaRocksDBPersentenceProviderTest { private static RocksDBPPPImpl rocksDBPersistenceProvider; - //private static String dbPath = "tmpdb", dbLogPath = "tmplogs"; - private static String dbPath = "mainnetpermadb", dbLogPath = "mainnetpermadb.log"; + private static String dbPath = "tmpdb", dbLogPath = "tmplogs"; + //private static String dbPath = "mainnetpermadb", dbLogPath = "mainnetpermadb.log"; @BeforeClass public static void setUpDb() throws Exception { From c28bbaabdcc66047f1101e92eb183f3c45e0c8be Mon Sep 17 00:00:00 2001 From: olaf Date: Wed, 4 Sep 2019 11:03:10 +0200 Subject: [PATCH 04/26] Removed the countered transaction model, just pinning, unpinning and isPinned is left. --- src/main/java/com/iota/iri/service/API.java | 54 ++------ .../java/com/iota/iri/service/ApiCommand.java | 6 +- .../service/dto/BooleanValuesRespones.java | 20 +++ .../storage/PermanentPersistenceProvider.java | 24 ++-- .../java/com/iota/iri/storage/Tangle.java | 23 ++-- .../iri/storage/rocksDB/RocksDBPPPImpl.java | 126 +++++------------- .../PermaRocksDBPersentenceProviderTest.java | 22 +-- 7 files changed, 106 insertions(+), 169 deletions(-) create mode 100644 src/main/java/com/iota/iri/service/dto/BooleanValuesRespones.java diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 0dd11f6dcc..b146b62226 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -199,9 +199,8 @@ public API(IotaConfig configuration, IXI ixi, TransactionRequester transactionRe //----- permanent storage------ commandRoute.put(ApiCommand.PIN_TRANSACTION_HASHES, pinTransactionHashes()); - commandRoute.put(ApiCommand.PINNED_TRANSACTIONS_COUNT, pinnedTransactionsCount()); - commandRoute.put(ApiCommand.INCREASE_TRANSACTIONS_COUNT, increaseTransactionsCount()); - commandRoute.put(ApiCommand.DECREASE_TRANSACTIONS_COUNT, decreaseTransactionsCount()); + commandRoute.put(ApiCommand.IS_PINNED_TRANSACTIONS_COUNT, isPinned()); + commandRoute.put(ApiCommand.UNPIN_TRANSACTIONS, unpinTransactionHashes()); commandRoute.put(ApiCommand.PIN_TRANSACTIONS_TRYTES, pinTransactionTrytes()); } @@ -346,32 +345,17 @@ private AbstractResponse pinTransactionHashStatement(List transactionsLi return AbstractResponse.createEmptyResponse(); } - @Document(name="pinnedTransactionsCount") - private AbstractResponse pinnedTransactionsCountStatement(List transactionsList) throws Exception { - long[] toReturn = new long[transactionsList.size()]; - for (int i=0; i< toReturn.length;i++) { - toReturn[i] = tangle.getTransactionStorageCounter(HashFactory.TRANSACTION.create(transactionsList.get(i))); - } - return LongValuesResponse.create(toReturn); - } - - @Document(name="increaseTransactionsCount") - private AbstractResponse increaseTransactionsCountStatement(List transactionsList) throws Exception { - Hash[] toReturn = new Hash[transactionsList.size()]; - for (int i=0; i< toReturn.length;i++) { - toReturn[i] = HashFactory.TRANSACTION.create(transactionsList.get(i)); - } - tangle.incrementTransactions(toReturn); - return AbstractResponse.createEmptyResponse(); + @Document(name="isPinned") + private AbstractResponse isPinnedStatement(List transactionsList) throws Exception { + final List transactions = transactionsList.stream().map(HashFactory.TRANSACTION::create).collect(Collectors.toList()); + return BooleanValuesRespones.create(tangle.isPinned(transactions)); } - @Document(name="decreaseTransactionsCount") - private AbstractResponse decreaseTransactionsCountStatement(List transactionsList) throws Exception { - Hash[] toReturn = new Hash[transactionsList.size()]; - for (int i=0; i< toReturn.length;i++) { - toReturn[i] = HashFactory.TRANSACTION.create(transactionsList.get(i)); + @Document(name="unpinTransactionHashes") + private AbstractResponse unpinTransactionHashesStatement(List transactionsList) throws Exception { + for(String txHash: transactionsList){ + tangle.unpinTransaction(HashFactory.TRANSACTION.create(txHash)); } - tangle.decrementTransactions(toReturn); return AbstractResponse.createEmptyResponse(); } @@ -1818,38 +1802,28 @@ private Function, AbstractResponse> pinTransactionHashes() { }; } - private Function, AbstractResponse> pinnedTransactionsCount() { + private Function, AbstractResponse> isPinned() { return request -> { final List txids = getParameterAsList(request,"hashes", HASH_SIZE); try { - return pinnedTransactionsCountStatement(txids); + return isPinnedStatement(txids); } catch (Exception e) { throw new IllegalStateException(e); } }; } - private Function, AbstractResponse> increaseTransactionsCount() { + private Function, AbstractResponse> unpinTransactionHashes() { return request -> { final List txids = getParameterAsList(request,"hashes", HASH_SIZE); try { - return increaseTransactionsCountStatement(txids); + return unpinTransactionHashesStatement(txids); } catch (Exception e) { throw new IllegalStateException(e); } }; } - private Function, AbstractResponse> decreaseTransactionsCount() { - return request -> { - final List txids = getParameterAsList(request,"hashes", HASH_SIZE); - try { - return decreaseTransactionsCountStatement(txids); - } catch (Exception e) { - throw new IllegalStateException(e); - } - }; - } // --------------------------- diff --git a/src/main/java/com/iota/iri/service/ApiCommand.java b/src/main/java/com/iota/iri/service/ApiCommand.java index a80822b8ca..38d62cbf57 100644 --- a/src/main/java/com/iota/iri/service/ApiCommand.java +++ b/src/main/java/com/iota/iri/service/ApiCommand.java @@ -104,11 +104,9 @@ public enum ApiCommand { */ PIN_TRANSACTION_HASHES("pinTransactionHashes"), - PINNED_TRANSACTIONS_COUNT("pinnedTransactionsCount"), + UNPIN_TRANSACTIONS("unpinTransactionHashes"), - INCREASE_TRANSACTIONS_COUNT("increaseTransactionsCount"), - - DECREASE_TRANSACTIONS_COUNT("decreaseTransactionsCount"), + IS_PINNED_TRANSACTIONS_COUNT("isPinned"), PIN_TRANSACTIONS_TRYTES("pinTransactionsTrytes"); diff --git a/src/main/java/com/iota/iri/service/dto/BooleanValuesRespones.java b/src/main/java/com/iota/iri/service/dto/BooleanValuesRespones.java new file mode 100644 index 0000000000..f378a35e66 --- /dev/null +++ b/src/main/java/com/iota/iri/service/dto/BooleanValuesRespones.java @@ -0,0 +1,20 @@ +package com.iota.iri.service.dto; + +public class BooleanValuesRespones extends AbstractResponse { + /** + * States of the specified addresses in Boolean + * Order of booleans is equal to order of the supplied addresses. + */ + private boolean[] values; + + + public static AbstractResponse create(boolean[] values) { + BooleanValuesRespones res = new BooleanValuesRespones(); + res.values = values; + return res; + } + + public boolean[] getValue() { + return this.values; + } +} \ No newline at end of file diff --git a/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java b/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java index 6c8fea4e43..18caa35600 100644 --- a/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java @@ -1,28 +1,26 @@ package com.iota.iri.storage; import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; import com.iota.iri.model.persistables.Hashes; +import java.util.List; + public interface PermanentPersistenceProvider { void init() throws Exception; boolean isAvailable(); void shutdown(); - boolean saveTransaction(TransactionViewModel model, Indexable index) throws Exception; - - void incrementTransactions(Indexable[] indexes) throws Exception; - - void decrementTransactions(Indexable[] indexes) throws Exception; - - long getCounter(Indexable index) throws Exception; + boolean pinTransaction(TransactionViewModel model, Hash index) throws Exception; + boolean unpinTransaction(Hash index) throws Exception; + boolean[] isPinned(List indexes) throws Exception; - boolean setCounter(Indexable index, long counter) throws Exception; - TransactionViewModel getTransaction(Indexable index) throws Exception; + TransactionViewModel getTransaction(Hash index) throws Exception; - Hashes findAddress(Indexable indexes) throws Exception; - Hashes findBundle(Indexable indexes) throws Exception; - Hashes findTag(Indexable indexes) throws Exception; - Hashes findApprovee(Indexable indexes) throws Exception; + Hashes findAddress(Hash indexes) throws Exception; + Hashes findBundle(Hash indexes) throws Exception; + Hashes findTag(Hash indexes) throws Exception; + Hashes findApprovee(Hash indexes) throws Exception; } diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index 3eb20ebc64..d0efe90f90 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -370,7 +370,7 @@ public void clearMetadata(Class column) throws Exception { public void pinTransaction(TransactionViewModel tvm) throws Exception { for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { - provider.saveTransaction(tvm, tvm.getHash()); + provider.pinTransaction(tvm, tvm.getHash()); } } @@ -382,29 +382,26 @@ public void pinTransaction(Hash hash) throws Exception { } } - public void incrementTransactions(Indexable[] indexes) throws Exception { + public void unpinTransaction(Hash hash) throws Exception { for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { - provider.incrementTransactions(indexes); + provider.unpinTransaction(hash); } } - public void decrementTransactions(Indexable[] indexes) throws Exception { + public boolean[] isPinned(List transactionHashes) throws Exception { + boolean[] mergedResult = new boolean[transactionHashes.size()]; for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { - provider.decrementTransactions(indexes); + boolean[] providerResult = provider.isPinned(transactionHashes); + for(int i = 0; i < mergedResult.length; i++){ + mergedResult[i] = (mergedResult[i] || providerResult[i]); + } } - } - public long getTransactionStorageCounter(Hash index) throws Exception { - long toReturn = 0; - for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { - toReturn = Math.max(0, provider.getCounter(index)); - } - return toReturn; + return mergedResult; } - // ------------------------------------------------- diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index 88c5975e17..57eff651bd 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -35,8 +35,7 @@ public class RocksDBPPPImpl implements PermanentPersistenceProvider, Persistence private final String logPath; private final int cacheSize; - public static String COUNTED_TRANSACTION = "counted-transaction"; - public static String COUNTER_INDEX = "counter-index"; + public static String TRANSACTION_COLUMN = "transaction"; public static String ADDRESS_INDEX = "address-index"; public static String BUNDLE_INDEX = "bundle-index"; @@ -115,9 +114,10 @@ public Set keysWithMissingReferences(Class modelClass, Class ot @Override public Persistable get(Class model, Indexable index) throws Exception { Persistable object = (Persistable) model.newInstance(); + Hash hashIndex = (Hash)index; if(object instanceof Transaction) { - TransactionViewModel tvm = getTransaction(index); + TransactionViewModel tvm = getTransaction(hashIndex); if(tvm == null){ return object; } @@ -137,22 +137,22 @@ public Persistable get(Class model, Indexable index) throws Exception { if(object instanceof Bundle){ Bundle toReturn = new Bundle(); - toReturn.set = findBundle(index).set; + toReturn.set = findBundle(hashIndex).set; return toReturn; } if(object instanceof Address){ Address toReturn = new Address(); - toReturn.set = findAddress(index).set; + toReturn.set = findAddress(hashIndex).set; return toReturn; } if(object instanceof Tag){ Tag toReturn = new Tag(); - toReturn.set = findTag(index).set; + toReturn.set = findTag(hashIndex).set; return toReturn; } if(object instanceof Approvee){ Approvee toReturn = new Approvee(); - toReturn.set = findApprovee(index).set; + toReturn.set = findApprovee(hashIndex).set; return toReturn; } return object; @@ -160,10 +160,6 @@ public Persistable get(Class model, Indexable index) throws Exception { public void loopAddressIndex(){ try (RocksIterator iterator = db.newIterator(columnMap.get(ADDRESS_INDEX))) { - - - - iterator.seekToFirst(); while(iterator.isValid()) { Hashes h = new Hashes(); @@ -240,22 +236,15 @@ public List loadAllKeysFromTable(Class model) { @Override - public boolean saveTransaction(TransactionViewModel model, Indexable index) throws Exception { - long counter = getCounter(index); - //check if the transaction does not exists - if(counter <= 0){ + public boolean pinTransaction(TransactionViewModel model, Hash index) throws Exception { + boolean exists = isPinned(Collections.singletonList(index))[0]; + if(!exists){ try (WriteBatch writeBatch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions()) { byte[] txBytes = model.getBytes(); - writeBatch.put(columnMap.get(COUNTED_TRANSACTION), index.bytes(), txBytes); - - //Prepare counted index - ByteBuffer counterBuffer = ByteBuffer.allocate(Long.BYTES); - counterBuffer.putLong(1L); - writeBatch.put(columnMap.get(COUNTER_INDEX),index.bytes(), counterBuffer.array()); + writeBatch.put(columnMap.get(TRANSACTION_COLUMN), index.bytes(), txBytes); - //addToIndex(writeBatch, columnMap.get(COUNTER_INDEX), counterBuffer.array() , index); addToIndex(writeBatch, columnMap.get(ADDRESS_INDEX), model.getAddressHash(), index); addToIndex(writeBatch, columnMap.get(TAG_INDEX), model.getTagValue(), index); addToIndex(writeBatch, columnMap.get(BUNDLE_INDEX), model.getBundleHash(), index); @@ -271,34 +260,33 @@ public boolean saveTransaction(TransactionViewModel model, Indexable index) thro return false; } - /** - * This is a dump method and does not take into account the mechanisms for storing - * @param indexes - * @return - * @throws Exception - */ - @Override - public void incrementTransactions(Indexable[] indexes) throws Exception { + public boolean unpinTransaction(Hash index) throws Exception { try (WriteBatch writeBatch = new WriteBatch(); - WriteOptions writeOptions = new WriteOptions()) { - for (Indexable i: indexes) { - //sorry no merge for long, the C++ lib has a merge function for Uin64 but that is not (natively)supported by Java. - long incrementedCounter = getCounter(i) + 1; - writeBatch.put(columnMap.get(COUNTER_INDEX), i.bytes(), ByteBuffer.allocate(Long.BYTES).putLong(incrementedCounter).array() ); - } + WriteOptions writeOptions = new WriteOptions()) { + safeDeleteTransaction(writeBatch, index); db.write(writeOptions, writeBatch); + } + return true; + } + @Override + public boolean[] isPinned(List indexes) throws Exception { + boolean[] result = new boolean[indexes.size()]; + ColumnFamilyHandle handle = columnMap.get(TRANSACTION_COLUMN); + for(int i = 0; i < result.length; i++){ + result[i] = db.keyMayExist(handle, indexes.get(i).bytes(), new StringBuilder()); } + return result; } + @VisibleForTesting - public void safeDeleteTransaction(WriteBatch writeBatch, Indexable key) throws Exception { + public void safeDeleteTransaction(WriteBatch writeBatch, Hash key) throws Exception { byte[] keyBytes = key.bytes(); TransactionViewModel tx = getTransaction(key); - writeBatch.delete(columnMap.get(COUNTER_INDEX), keyBytes); - writeBatch.delete(columnMap.get(COUNTED_TRANSACTION), keyBytes); + writeBatch.delete(columnMap.get(TRANSACTION_COLUMN), keyBytes); removeFromIndex(writeBatch, columnMap.get(ADDRESS_INDEX), tx.getAddressHash(), key); removeFromIndex(writeBatch, columnMap.get(TAG_INDEX), tx.getTagValue(), key); @@ -377,54 +365,10 @@ public int indexOf(byte[] outerArray, byte[] smallerArray) { } - - @Override - public void decrementTransactions(Indexable[] indexes) throws Exception { - try (WriteBatch writeBatch = new WriteBatch(); - WriteOptions writeOptions = new WriteOptions()) { - for (Indexable i: indexes) { - //sorry no merge for long, the C++ lib has a merge function for Uin64 but that is not (natively)supported by Java. - long decrementedCounter = getCounter(i) -1; - if(decrementedCounter <= 0){ - safeDeleteTransaction(writeBatch, i); - }else { - writeBatch.put(columnMap.get(COUNTER_INDEX), i.bytes(), ByteBuffer.allocate(Long.BYTES).putLong(decrementedCounter).array()); - } - } - - db.write(writeOptions, writeBatch); - } - } - - @Override - public boolean setCounter(Indexable index, long counter) throws Exception { - try (WriteBatch writeBatch = new WriteBatch(); - WriteOptions writeOptions = new WriteOptions()) { - if(counter <= 0){ - safeDeleteTransaction(writeBatch, index); - }else { - writeBatch.put(columnMap.get(COUNTER_INDEX), index.bytes(), ByteBuffer.allocate(Long.BYTES).putLong(counter).array()); - } - db.write(writeOptions, writeBatch); - } - return false; - } - - @Override - public long getCounter(Indexable index) throws Exception { - if(index == null){return -1;} - byte[] dbResult = db.get(columnMap.get(COUNTER_INDEX), index.bytes()); - if(dbResult != null && dbResult.length == Long.BYTES) { - long counter = ByteBuffer.wrap(dbResult).getLong(); - return counter; - } - return 0; - } - @Override - public TransactionViewModel getTransaction(Indexable index) throws Exception { + public TransactionViewModel getTransaction(Hash index) throws Exception { if(index == null){return null;} - byte[] dbResult = db.get(columnMap.get(COUNTED_TRANSACTION), index.bytes()); + byte[] dbResult = db.get(columnMap.get(TRANSACTION_COLUMN), index.bytes()); if(dbResult != null && dbResult.length > 0){ return new TransactionViewModel(TransactionViewModel.trits(dbResult), Hash.NULL_HASH); } @@ -432,19 +376,19 @@ public TransactionViewModel getTransaction(Indexable index) throws Exception { } @Override - public Hashes findAddress(Indexable index) throws Exception { + public Hashes findAddress(Hash index) throws Exception { return getIndex(columnMap.get(ADDRESS_INDEX), index); } @Override - public Hashes findBundle(Indexable index) throws Exception { + public Hashes findBundle(Hash index) throws Exception { return getIndex(columnMap.get(BUNDLE_INDEX), index); } @Override - public Hashes findTag(Indexable index) throws Exception { + public Hashes findTag(Hash index) throws Exception { return getIndex(columnMap.get(TAG_INDEX), index); } @Override - public Hashes findApprovee(Indexable index) throws Exception { + public Hashes findApprovee(Hash index) throws Exception { return getIndex(columnMap.get(APPROVEE_INDEX), index); } @@ -522,8 +466,8 @@ private void initDB(String path, String logPath) throws Exception { //Add default column family. Main motivation is to not change legacy code columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions)); - columnFamilyDescriptors.add(new ColumnFamilyDescriptor(COUNTED_TRANSACTION.getBytes("UTF-8"), columnFamilyOptions)); - columnFamilyDescriptors.add(new ColumnFamilyDescriptor(COUNTER_INDEX.getBytes("UTF-8"), columnFamilyOptions)); + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(TRANSACTION_COLUMN.getBytes("UTF-8"), columnFamilyOptions)); + // columnFamilyDescriptors.add(new ColumnFamilyDescriptor(COUNTER_INDEX.getBytes("UTF-8"), columnFamilyOptions)); columnFamilyDescriptors.add(new ColumnFamilyDescriptor(BUNDLE_INDEX.getBytes("UTF-8"), columnFamilyOptions)); columnFamilyDescriptors.add(new ColumnFamilyDescriptor(TAG_INDEX.getBytes("UTF-8"), columnFamilyOptions)); columnFamilyDescriptors.add(new ColumnFamilyDescriptor(ADDRESS_INDEX.getBytes("UTF-8"), columnFamilyOptions)); diff --git a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java index a4bef80097..e01d1ecbe4 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java @@ -4,8 +4,11 @@ import com.iota.iri.model.Hash; import com.iota.iri.storage.Indexable; +import org.apache.commons.io.FileUtils; import org.junit.*; +import java.io.File; +import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -28,9 +31,9 @@ public static void setUpDb() throws Exception { @AfterClass public static void destroyDb() { rocksDBPersistenceProvider.shutdown(); - //FileUtils.deleteQuietly(new File(dbPath)); - //FileUtils.deleteQuietly(new File(dbLogPath)); - //rocksDBPersistenceProvider = null; + FileUtils.deleteQuietly(new File(dbPath)); + FileUtils.deleteQuietly(new File(dbLogPath)); + rocksDBPersistenceProvider = null; } @Before @@ -69,14 +72,14 @@ public void testSelectiveTransactionModel() throws Exception { for(TransactionViewModel v: mm){ System.out.println(v.getHash() + " \t " + v.getAddressHash()); - rocksDBPersistenceProvider.saveTransaction(v, v.getHash()); + rocksDBPersistenceProvider.pinTransaction(v, v.getHash()); } - rocksDBPersistenceProvider.incrementTransactions(new Indexable[]{transaction.getHash()}); - long count = rocksDBPersistenceProvider.getCounter(transaction.getHash()); - Assert.assertEquals(count, 2); + //rocksDBPersistenceProvider.incrementTransactions(new Indexable[]{transaction.getHash()}); + boolean isPinned = rocksDBPersistenceProvider.isPinned(Collections.singletonList(transaction.getHash()))[0]; + Assert.assertTrue(isPinned); //See if all indexes are updated. Assert.assertTrue(rocksDBPersistenceProvider.findAddress(transaction2.getAddressHash()).set.contains(transaction2.getHash())); @@ -86,7 +89,10 @@ public void testSelectiveTransactionModel() throws Exception { Assert.assertTrue(rocksDBPersistenceProvider.findApprovee(transaction2.getBranchTransactionHash()).set.contains(transaction2.getHash())); //Test decremeant delete; - rocksDBPersistenceProvider.decrementTransactions(new Indexable[]{transaction3.getHash()}); + rocksDBPersistenceProvider.unpinTransaction(transaction3.getHash()); + + boolean isPinnedTwo = rocksDBPersistenceProvider.isPinned(Collections.singletonList(transaction3.getHash()))[0]; + Assert.assertTrue(!isPinnedTwo); Assert.assertNull(rocksDBPersistenceProvider.getTransaction(transaction3.getHash())); } From e27f1e721cb1124c5891a066c3aff3a3c66e398b Mon Sep 17 00:00:00 2001 From: olaf Date: Mon, 9 Sep 2019 09:49:11 +0200 Subject: [PATCH 05/26] Renaming and comments --- src/main/java/com/iota/iri/service/API.java | 33 ++++++++++++++++--- .../java/com/iota/iri/service/ApiCommand.java | 18 +++++++--- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index b146b62226..3e7be98258 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -197,11 +197,11 @@ public API(IotaConfig configuration, IXI ixi, TransactionRequester transactionRe commandRoute.put(ApiCommand.CHECK_CONSISTENCY, checkConsistency()); commandRoute.put(ApiCommand.WERE_ADDRESSES_SPENT_FROM, wereAddressesSpentFrom()); - //----- permanent storage------ commandRoute.put(ApiCommand.PIN_TRANSACTION_HASHES, pinTransactionHashes()); + commandRoute.put(ApiCommand.PIN_TRANSACTIONS_TRYTES, pinTransactionTrytes()); commandRoute.put(ApiCommand.IS_PINNED_TRANSACTIONS_COUNT, isPinned()); commandRoute.put(ApiCommand.UNPIN_TRANSACTIONS, unpinTransactionHashes()); - commandRoute.put(ApiCommand.PIN_TRANSACTIONS_TRYTES, pinTransactionTrytes()); + } @@ -324,18 +324,29 @@ private AbstractResponse wereAddressesSpentFromStatement(List addresses) // ----------- permanent storage -------- + /** + * Pins transactions based on transaction trytes. It will store the transaction trytes in permanent storage. + * @param trytes list of transaction trytes. + * @return list of booleans if it the pinning was successful or not. + * @throws Exception + */ @Document(name="pinTransactionTrytes") public AbstractResponse pinTransactionTrytesStatement(List trytes) throws Exception { final List elements = convertTrytes(trytes); for (final TransactionViewModel transactionViewModel : elements) { - //store transactions tangle.pinTransaction(transactionViewModel); } return AbstractResponse.createEmptyResponse(); } + /** + * Pins transactions that are already in the current database. It will move those transactions to permanent storage. + * @param transactionsList + * @return List of booleans that represent of the transaction transfer was successful. + * @throws Exception + */ @Document(name="pinTransactionHashes") - private AbstractResponse pinTransactionHashStatement(List transactionsList) throws Exception { + private AbstractResponse pinTransactionHashesStatement(List transactionsList) throws Exception { final List transactions = transactionsList.stream().map(HashFactory.TRANSACTION::create).collect(Collectors.toList()); for (Hash transaction : transactions) { @@ -345,12 +356,24 @@ private AbstractResponse pinTransactionHashStatement(List transactionsLi return AbstractResponse.createEmptyResponse(); } + /** + * Checks if transactions are pinned + * @param transactionsList list of transaction hashes to check. + * @return List of booleans, true equals it is pinned and false for not pinned. + * @throws Exception + */ @Document(name="isPinned") private AbstractResponse isPinnedStatement(List transactionsList) throws Exception { final List transactions = transactionsList.stream().map(HashFactory.TRANSACTION::create).collect(Collectors.toList()); return BooleanValuesRespones.create(tangle.isPinned(transactions)); } + /** + * Unpins a transaction, it will be removed from permanent storage. + * @param transactionsList List of transaction ID's to be unpinned. + * @return An empty response with the time it took to unpin. + * @throws Exception + */ @Document(name="unpinTransactionHashes") private AbstractResponse unpinTransactionHashesStatement(List transactionsList) throws Exception { for(String txHash: transactionsList){ @@ -1795,7 +1818,7 @@ private Function, AbstractResponse> pinTransactionHashes() { return request -> { final List txids = getParameterAsList(request,"hashes", HASH_SIZE); try { - return pinTransactionHashStatement(txids); + return pinTransactionHashesStatement(txids); } catch (Exception e) { throw new IllegalStateException(e); } diff --git a/src/main/java/com/iota/iri/service/ApiCommand.java b/src/main/java/com/iota/iri/service/ApiCommand.java index 38d62cbf57..776dda7bbb 100644 --- a/src/main/java/com/iota/iri/service/ApiCommand.java +++ b/src/main/java/com/iota/iri/service/ApiCommand.java @@ -97,18 +97,26 @@ public enum ApiCommand { */ WERE_ADDRESSES_SPENT_FROM("wereAddressesSpentFrom"), - // ---------- permanent storage /** - * Pins a transaction + * Pins a transaction from normal storage identified by their hash */ PIN_TRANSACTION_HASHES("pinTransactionHashes"), - UNPIN_TRANSACTIONS("unpinTransactionHashes"), + /** + * Pins a transaction based on sent trytes + */ + PIN_TRANSACTIONS_TRYTES("pinTransactionsTrytes"), - IS_PINNED_TRANSACTIONS_COUNT("isPinned"), + /** + * Unpins a transaction + */ + UNPIN_TRANSACTIONS("unpinTransactionHashes"), - PIN_TRANSACTIONS_TRYTES("pinTransactionsTrytes"); + /** + * Checks if a transaction is pinned or not + */ + IS_PINNED_TRANSACTIONS_COUNT("isPinned"); From c9894701382f2130eaf2ba590c5b9691b6287803 Mon Sep 17 00:00:00 2001 From: olaf Date: Mon, 30 Sep 2019 12:11:18 +0200 Subject: [PATCH 06/26] Fix for RocksDBPPPImpl to handle integer indexes. --- .../iota/iri/model/persistables/SpentAddress.java | 12 ------------ src/main/java/com/iota/iri/storage/Tangle.java | 2 +- .../iota/iri/storage/rocksDB/RocksDBPPPImpl.java | 13 +++++++------ 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/iota/iri/model/persistables/SpentAddress.java b/src/main/java/com/iota/iri/model/persistables/SpentAddress.java index 4ef76af9f9..e6798f6fda 100644 --- a/src/main/java/com/iota/iri/model/persistables/SpentAddress.java +++ b/src/main/java/com/iota/iri/model/persistables/SpentAddress.java @@ -31,26 +31,14 @@ public boolean canMerge() { return false; } - @Override public Persistable mergeInto(Persistable source) throws OperationNotSupportedException { throw new OperationNotSupportedException("This object is not mergeable"); } - @Override public boolean exists() { return exists; } - @Override - public Persistable mergeTwo(Persistable nrTwo){ - return null; - } - - @Override - public boolean isEmpty() { - return exists; - } - } diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index 4ef2bfcd74..330ed03fd4 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -386,7 +386,7 @@ public void pinTransaction(TransactionViewModel tvm) throws Exception { public void pinTransaction(Hash hash) throws Exception { Transaction tx = (Transaction)load(Transaction.class, hash); - if(!tx.isEmpty()) { + if(!tx.exists()) { TransactionViewModel tvm = new TransactionViewModel(tx, hash); pinTransaction(tvm); } diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index 57eff651bd..f54383f1f7 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -114,10 +114,11 @@ public Set keysWithMissingReferences(Class modelClass, Class ot @Override public Persistable get(Class model, Indexable index) throws Exception { Persistable object = (Persistable) model.newInstance(); - Hash hashIndex = (Hash)index; + if(object instanceof Transaction) { - TransactionViewModel tvm = getTransaction(hashIndex); + + TransactionViewModel tvm = getTransaction((Hash)index); if(tvm == null){ return object; } @@ -137,22 +138,22 @@ public Persistable get(Class model, Indexable index) throws Exception { if(object instanceof Bundle){ Bundle toReturn = new Bundle(); - toReturn.set = findBundle(hashIndex).set; + toReturn.set = findBundle((Hash)index).set; return toReturn; } if(object instanceof Address){ Address toReturn = new Address(); - toReturn.set = findAddress(hashIndex).set; + toReturn.set = findAddress((Hash)index).set; return toReturn; } if(object instanceof Tag){ Tag toReturn = new Tag(); - toReturn.set = findTag(hashIndex).set; + toReturn.set = findTag((Hash)index).set; return toReturn; } if(object instanceof Approvee){ Approvee toReturn = new Approvee(); - toReturn.set = findApprovee(hashIndex).set; + toReturn.set = findApprovee((Hash)index).set; return toReturn; } return object; From be9fd593b51b006d56f3932d62a300a913ab167a Mon Sep 17 00:00:00 2001 From: olaf Date: Mon, 30 Sep 2019 13:17:25 +0200 Subject: [PATCH 07/26] Name changes and comments --- .../com/iota/iri/conf/BaseIotaConfig.java | 8 +-- src/main/java/com/iota/iri/service/API.java | 2 +- .../java/com/iota/iri/service/Feature.java | 25 ++++++-- ...spones.java => BooleanValuesResponse.java} | 12 ++-- .../storage/PermanentPersistenceProvider.java | 63 ++++++++++++++++--- .../iri/storage/rocksDB/RocksDBPPPImpl.java | 32 +--------- .../PermaRocksDBPersentenceProviderTest.java | 24 ++----- 7 files changed, 90 insertions(+), 76 deletions(-) rename src/main/java/com/iota/iri/service/dto/{BooleanValuesRespones.java => BooleanValuesResponse.java} (52%) diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index f14f9856c2..208af8349e 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -390,12 +390,7 @@ protected void setIxiDir(String ixiDir) { this.ixiDir = ixiDir; } - - - //------- PERMADB ------------- - - @Override public boolean isSelectivePermaEnabled() { return this.permaDbEnabled;} @@ -470,9 +465,8 @@ public boolean permaIsRescanDb() { protected void setPermaRescanDb(boolean permaRescanDb) { this.permaRescanDb = permaRescanDb; } - - //----------------------------- + @Override public String getDbPath() { return dbPath; diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 3e7be98258..9d128e999d 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -365,7 +365,7 @@ private AbstractResponse pinTransactionHashesStatement(List transactions @Document(name="isPinned") private AbstractResponse isPinnedStatement(List transactionsList) throws Exception { final List transactions = transactionsList.stream().map(HashFactory.TRANSACTION::create).collect(Collectors.toList()); - return BooleanValuesRespones.create(tangle.isPinned(transactions)); + return BooleanValuesResponse.create(tangle.isPinned(transactions)); } /** diff --git a/src/main/java/com/iota/iri/service/Feature.java b/src/main/java/com/iota/iri/service/Feature.java index 0c78d2f7d1..f90a142090 100644 --- a/src/main/java/com/iota/iri/service/Feature.java +++ b/src/main/java/com/iota/iri/service/Feature.java @@ -39,7 +39,12 @@ public enum Feature { /** * This node has the zero message queue enabled for fetching/reading "activities" on the node */ - ZMQ("zeroMessageQueue"); + ZMQ("zeroMessageQueue"), + + /** + * This node can do transaction pinning + */ + TRANSACTION_PINNING("transactionPinning"); private String name; @@ -72,16 +77,26 @@ public static Feature[] calculateFeatures(IotaConfig configuration) { if (configuration.isZmqEnabled()) { features.add(ZMQ); } + if (configuration.isSelectivePermaEnabled()) { + features.add(TRANSACTION_PINNING); + } List apiFeatures = new ArrayList(Arrays.asList(new Feature[] { PROOF_OF_WORK })); - + for (String disabled : configuration.getRemoteLimitApi()) { + switch (disabled) { - case "attachToTangle": - apiFeatures.remove(PROOF_OF_WORK); - break; + case "attachToTangle": + apiFeatures.remove(PROOF_OF_WORK); + break; + case "pinTransactionHashes": + apiFeatures.remove(TRANSACTION_PINNING); + break; + case "pinTransactionsTrytes": + apiFeatures.remove(TRANSACTION_PINNING); + break; default: break; diff --git a/src/main/java/com/iota/iri/service/dto/BooleanValuesRespones.java b/src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java similarity index 52% rename from src/main/java/com/iota/iri/service/dto/BooleanValuesRespones.java rename to src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java index f378a35e66..0764f6a8db 100644 --- a/src/main/java/com/iota/iri/service/dto/BooleanValuesRespones.java +++ b/src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java @@ -1,20 +1,20 @@ package com.iota.iri.service.dto; -public class BooleanValuesRespones extends AbstractResponse { +public class BooleanValuesResponse extends AbstractResponse { /** * States of the specified addresses in Boolean * Order of booleans is equal to order of the supplied addresses. */ - private boolean[] values; + private boolean[] result; public static AbstractResponse create(boolean[] values) { - BooleanValuesRespones res = new BooleanValuesRespones(); - res.values = values; + BooleanValuesResponse res = new BooleanValuesResponse(); + res.result = values; return res; } - public boolean[] getValue() { - return this.values; + public boolean[] getResult() { + return this.result; } } \ No newline at end of file diff --git a/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java b/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java index 18caa35600..c1da6793ac 100644 --- a/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java @@ -9,18 +9,65 @@ public interface PermanentPersistenceProvider { - void init() throws Exception; - boolean isAvailable(); - void shutdown(); + /** + * Pins a {@code model} for permanent storage. + * @param model the transaction to store. + * @param index the Hash for indexing + * @return true of succesfully pinnend + * @throws Exception + */ boolean pinTransaction(TransactionViewModel model, Hash index) throws Exception; + + /*** + * Unpins a transaction, removing it from permanent storage + * @param index the transaction ID + * @return true of succesfully unpinnend + * @throws Exception + */ boolean unpinTransaction(Hash index) throws Exception; - boolean[] isPinned(List indexes) throws Exception; + /*** + * Checks if transactions are pinnend or not + * @param indexes + * @return List of booleans in order of the list of hashes given. True is pinned, false is not pinned + * @throws Exception + */ + boolean[] isPinned(List indexes) throws Exception; + /*** + * Retrieves the transactions from the permanent storage + * @param index the transaction ID to retrieve + * @return The transactions + * @throws Exception + */ TransactionViewModel getTransaction(Hash index) throws Exception; - Hashes findAddress(Hash indexes) throws Exception; - Hashes findBundle(Hash indexes) throws Exception; - Hashes findTag(Hash indexes) throws Exception; - Hashes findApprovee(Hash indexes) throws Exception; + /*** + * Finds a list of transaction ID's related to the address hash + * @param index address hash + * @return List of found transaction ID's + * @throws Exception + */ + Hashes findAddress(Hash index) throws Exception; + /*** + * Finds a list of transaction ID's related to the bundle hash + * @param index bundle hash + * @return List of found transaction ID's + * @throws Exception + */ + Hashes findBundle(Hash index) throws Exception; + /*** + * Finds a list of transaction ID's related to the tag + * @param index tag + * @return List of found transaction ID's + * @throws Exception + */ + Hashes findTag(Hash index) throws Exception; + /*** + * Finds a list of approvee's related to the transaction ID + * @param index transaction ID + * @return List of found approvee transaction ID's + * @throws Exception + */ + Hashes findApprovee(Hash index) throws Exception; } diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index f54383f1f7..e6b7d4478b 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -38,7 +38,6 @@ public class RocksDBPPPImpl implements PermanentPersistenceProvider, Persistence public static String TRANSACTION_COLUMN = "transaction"; public static String ADDRESS_INDEX = "address-index"; public static String BUNDLE_INDEX = "bundle-index"; - public static String TAG_INDEX = "tag-index"; public static String APPROVEE_INDEX = "approvee-index"; @@ -55,7 +54,6 @@ public RocksDBPPPImpl(String dbPath, String logPath, int cacheSize) { this.dbPath = dbPath; this.logPath = logPath; this.cacheSize = cacheSize; - } @Override @@ -79,7 +77,6 @@ public void shutdown() { IotaIOUtils.closeQuietly(db, options, bloomFilter); } - //----------- PersistenceProvider only overrides --------------- @Override public boolean save(Persistable model, Indexable index) throws Exception { @@ -115,7 +112,6 @@ public Set keysWithMissingReferences(Class modelClass, Class ot public Persistable get(Class model, Indexable index) throws Exception { Persistable object = (Persistable) model.newInstance(); - if(object instanceof Transaction) { TransactionViewModel tvm = getTransaction((Hash)index); @@ -159,21 +155,6 @@ public Persistable get(Class model, Indexable index) throws Exception { return object; } - public void loopAddressIndex(){ - try (RocksIterator iterator = db.newIterator(columnMap.get(ADDRESS_INDEX))) { - iterator.seekToFirst(); - while(iterator.isValid()) { - Hashes h = new Hashes(); - byte[] value = iterator.value(); - h.read(value); - Hash i = HashFactory.ADDRESS.create(iterator.value()); - log.info(i.toString() + " " + h.toString()); - iterator.next(); - } - - } - } - @Override public boolean mayExist(Class model, Indexable index) throws Exception { return false; @@ -249,8 +230,6 @@ public boolean pinTransaction(TransactionViewModel model, Hash index) throws Exc addToIndex(writeBatch, columnMap.get(ADDRESS_INDEX), model.getAddressHash(), index); addToIndex(writeBatch, columnMap.get(TAG_INDEX), model.getTagValue(), index); addToIndex(writeBatch, columnMap.get(BUNDLE_INDEX), model.getBundleHash(), index); - - //TODO only add when relevant in reference. addToIndex(writeBatch, columnMap.get(APPROVEE_INDEX), model.getTrunkTransactionHash(), index); addToIndex(writeBatch, columnMap.get(APPROVEE_INDEX), model.getBranchTransactionHash(), index); db.write(writeOptions, writeBatch); @@ -315,16 +294,12 @@ public void removeFromIndex(WriteBatch writeBatch, ColumnFamilyHandle column, In buffer.put(delimiter); buffer.put(subarray2); } - - writeBatch.put(column, key.bytes(), buffer.array()); } - } - - } } + @VisibleForTesting public void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) throws Exception{ byte[] indexBytes = indexValue.bytes(); @@ -337,7 +312,6 @@ public void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexab if(keyLoc == -1){//not found ByteBuffer buffer = ByteBuffer.allocate(dbResult.length + indexBytes.length + (dbResult.length > 0 ? 1 : 0));//+1 delimiter length - buffer.put(dbResult); if(dbResult.length > 0){ //Means we add on the end of the current stream. @@ -345,7 +319,6 @@ public void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexab buffer.put(delimiter); } buffer.put(indexBytes); - writeBatch.put(column, key.bytes(), buffer.array()); } } @@ -474,9 +447,6 @@ private void initDB(String path, String logPath) throws Exception { columnFamilyDescriptors.add(new ColumnFamilyDescriptor(ADDRESS_INDEX.getBytes("UTF-8"), columnFamilyOptions)); columnFamilyDescriptors.add(new ColumnFamilyDescriptor(APPROVEE_INDEX.getBytes("UTF-8"), columnFamilyOptions)); - - - db = RocksDB.open(options, path, columnFamilyDescriptors, columnFamilyHandles); db.enableFileDeletions(true); diff --git a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java index e01d1ecbe4..33b1124fed 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java @@ -19,7 +19,6 @@ public class PermaRocksDBPersentenceProviderTest { private static RocksDBPPPImpl rocksDBPersistenceProvider; private static String dbPath = "tmpdb", dbLogPath = "tmplogs"; - //private static String dbPath = "mainnetpermadb", dbLogPath = "mainnetpermadb.log"; @BeforeClass public static void setUpDb() throws Exception { @@ -36,18 +35,6 @@ public static void destroyDb() { rocksDBPersistenceProvider = null; } - @Before - public void setUp() throws Exception { - - } - - @Test - public void readData() throws Exception { - rocksDBPersistenceProvider.loopAddressIndex(); - - - } - @Test public void testSelectiveTransactionModel() throws Exception { @@ -71,12 +58,9 @@ public void testSelectiveTransactionModel() throws Exception { mm.add(transaction4); for(TransactionViewModel v: mm){ - System.out.println(v.getHash() + " \t " + v.getAddressHash()); + //System.out.println(v.getHash() + " \t " + v.getAddressHash()); rocksDBPersistenceProvider.pinTransaction(v, v.getHash()); } - - - //rocksDBPersistenceProvider.incrementTransactions(new Indexable[]{transaction.getHash()}); boolean isPinned = rocksDBPersistenceProvider.isPinned(Collections.singletonList(transaction.getHash()))[0]; Assert.assertTrue(isPinned); @@ -91,8 +75,12 @@ public void testSelectiveTransactionModel() throws Exception { //Test decremeant delete; rocksDBPersistenceProvider.unpinTransaction(transaction3.getHash()); + //Test if indexes are updated after delete + Assert.assertFalse(rocksDBPersistenceProvider.findAddress(transaction3.getAddressHash()).set.contains(transaction3.getHash())); + Assert.assertFalse(rocksDBPersistenceProvider.findTag(transaction3.getTagValue()).set.contains(transaction3.getHash())); + boolean isPinnedTwo = rocksDBPersistenceProvider.isPinned(Collections.singletonList(transaction3.getHash()))[0]; - Assert.assertTrue(!isPinnedTwo); + Assert.assertFalse(isPinnedTwo); Assert.assertNull(rocksDBPersistenceProvider.getTransaction(transaction3.getHash())); } From 5d5d4634d8ec52b2a04fba4b58e51542e4e05960 Mon Sep 17 00:00:00 2001 From: olaf Date: Mon, 30 Sep 2019 14:25:19 +0200 Subject: [PATCH 08/26] Added boolean return values for transactionPinning --- src/main/java/com/iota/iri/service/API.java | 16 ++++++------- .../iri/service/dto/LongValuesResponse.java | 20 ---------------- .../java/com/iota/iri/storage/Tangle.java | 23 ++++++++++++++----- .../iri/storage/rocksDB/RocksDBPPPImpl.java | 9 ++++---- 4 files changed, 30 insertions(+), 38 deletions(-) delete mode 100644 src/main/java/com/iota/iri/service/dto/LongValuesResponse.java diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 9d128e999d..ae8b9b5dfe 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -333,10 +333,11 @@ private AbstractResponse wereAddressesSpentFromStatement(List addresses) @Document(name="pinTransactionTrytes") public AbstractResponse pinTransactionTrytesStatement(List trytes) throws Exception { final List elements = convertTrytes(trytes); - for (final TransactionViewModel transactionViewModel : elements) { - tangle.pinTransaction(transactionViewModel); + boolean[] result = new boolean[elements.size()]; + for(int i = 0; i < elements.size(); i++){ + result[i] = tangle.pinTransaction(elements.get(i)); } - return AbstractResponse.createEmptyResponse(); + return BooleanValuesResponse.create(result); } /** @@ -348,12 +349,11 @@ public AbstractResponse pinTransactionTrytesStatement(List trytes) throw @Document(name="pinTransactionHashes") private AbstractResponse pinTransactionHashesStatement(List transactionsList) throws Exception { final List transactions = transactionsList.stream().map(HashFactory.TRANSACTION::create).collect(Collectors.toList()); - - for (Hash transaction : transactions) { - tangle.pinTransaction(transaction); + boolean[] result = new boolean[transactions.size()]; + for(int i = 0; i < transactions.size(); i++){ + result[i] = tangle.pinTransaction(transactions.get(i)); } - - return AbstractResponse.createEmptyResponse(); + return BooleanValuesResponse.create(result); } /** diff --git a/src/main/java/com/iota/iri/service/dto/LongValuesResponse.java b/src/main/java/com/iota/iri/service/dto/LongValuesResponse.java deleted file mode 100644 index 6f084730c3..0000000000 --- a/src/main/java/com/iota/iri/service/dto/LongValuesResponse.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.iota.iri.service.dto; - -public class LongValuesResponse extends AbstractResponse { - /** - * States of the specified addresses in Boolean - * Order of booleans is equal to order of the supplied addresses. - */ - private long[] values; - - - public static AbstractResponse create(long[] values) { - LongValuesResponse res = new LongValuesResponse(); - res.values = values; - return res; - } - - public long[] getValue() { - return this.values; - } -} diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index 330ed03fd4..cf445095ab 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -378,24 +378,35 @@ public void clearMetadata(Class column) throws Exception { // ---------- Permanent storage capabilities -------- - public void pinTransaction(TransactionViewModel tvm) throws Exception { + /** + * @see PermanentPersistenceProvider#pinTransaction(TransactionViewModel, Hash) + */ + public boolean pinTransaction(TransactionViewModel tvm) throws Exception { + boolean success = false; for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { - provider.pinTransaction(tvm, tvm.getHash()); + success = provider.pinTransaction(tvm, tvm.getHash()) || success ; } + return success ; } - public void pinTransaction(Hash hash) throws Exception { + /** + * Retrieves transaction from persistence providers and then calls pinTransaction(TransactionViewModel, Hash) + */ + public boolean pinTransaction(Hash hash) throws Exception { Transaction tx = (Transaction)load(Transaction.class, hash); if(!tx.exists()) { TransactionViewModel tvm = new TransactionViewModel(tx, hash); - pinTransaction(tvm); + return pinTransaction(tvm); } + return false; } - public void unpinTransaction(Hash hash) throws Exception { + public boolean unpinTransaction(Hash hash) throws Exception { + boolean success = false; for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { - provider.unpinTransaction(hash); + success = provider.unpinTransaction(hash) || success; } + return success; } public boolean[] isPinned(List transactionHashes) throws Exception { diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index e6b7d4478b..2a26af9a72 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -233,11 +233,12 @@ public boolean pinTransaction(TransactionViewModel model, Hash index) throws Exc addToIndex(writeBatch, columnMap.get(APPROVEE_INDEX), model.getTrunkTransactionHash(), index); addToIndex(writeBatch, columnMap.get(APPROVEE_INDEX), model.getBranchTransactionHash(), index); db.write(writeOptions, writeBatch); + return true; } - - + }else{ + return true; //Already pinned, is fine to return true } - return false; + } @Override @@ -247,8 +248,8 @@ public boolean unpinTransaction(Hash index) throws Exception { safeDeleteTransaction(writeBatch, index); db.write(writeOptions, writeBatch); + return true; } - return true; } @Override From fe988f7ec248724054bd96d3e7240839266b39b6 Mon Sep 17 00:00:00 2001 From: olaf Date: Mon, 30 Sep 2019 14:26:41 +0200 Subject: [PATCH 09/26] Revert test code to original code --- .../iri/storage/rocksDB/RocksDBPersistenceProviderTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java index 5338fad97b..82d5189539 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java @@ -44,15 +44,13 @@ public void setUp() throws Exception { @Test public void testDeleteBatch() throws Exception { - Transaction tx = new Transaction(); + Persistable tx = new Transaction(); byte[] bytes = new byte[Transaction.SIZE]; Arrays.fill(bytes, (byte) 1); tx.read(bytes); tx.readMetadata(bytes); - - List> models = IntStream.range(1, 1000) - .mapToObj(i -> new Pair<>((Indexable) new IntegerIndex(i), (Persistable)tx)) + .mapToObj(i -> new Pair<>((Indexable) new IntegerIndex(i), tx)) .collect(Collectors.toList()); rocksDBPersistenceProvider.saveBatch(models); From 4b21d42daff65fa6fa2f3fe818e1872dbf520fdc Mon Sep 17 00:00:00 2001 From: olaf Date: Tue, 1 Oct 2019 00:33:28 +0200 Subject: [PATCH 10/26] Space removal --- .../iri/storage/rocksDB/RocksDBPersistenceProviderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java index 82d5189539..b0f9b49b32 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java @@ -44,7 +44,7 @@ public void setUp() throws Exception { @Test public void testDeleteBatch() throws Exception { - Persistable tx = new Transaction(); + Persistable tx = new Transaction(); byte[] bytes = new byte[Transaction.SIZE]; Arrays.fill(bytes, (byte) 1); tx.read(bytes); From 322764a27be22dfa5f62f102be0b648f72ae3e53 Mon Sep 17 00:00:00 2001 From: olaf Date: Tue, 1 Oct 2019 01:04:48 +0200 Subject: [PATCH 11/26] Braces fix --- src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index 2a26af9a72..3072fe526e 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -334,7 +334,7 @@ public int indexOf(byte[] outerArray, byte[] smallerArray) { break; } } - if (found) return i; + if (found) {return i;} } return -1; } From 300ec78677043260333a8a3cc8b48d6aef193fd9 Mon Sep 17 00:00:00 2001 From: olaf Date: Tue, 1 Oct 2019 11:20:52 +0200 Subject: [PATCH 12/26] Style updates and comments --- .../java/com/iota/iri/storage/Tangle.java | 8 +- .../iri/storage/rocksDB/RocksDBPPPImpl.java | 344 ++++++++++++------ .../PermaRocksDBPersentenceProviderTest.java | 6 +- 3 files changed, 238 insertions(+), 120 deletions(-) diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index cf445095ab..f01d1d49c1 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -400,7 +400,9 @@ public boolean pinTransaction(Hash hash) throws Exception { } return false; } - + /** + * @see PermanentPersistenceProvider#unpinTransaction(Hash) + */ public boolean unpinTransaction(Hash hash) throws Exception { boolean success = false; for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { @@ -408,7 +410,9 @@ public boolean unpinTransaction(Hash hash) throws Exception { } return success; } - + /** + * @see PermanentPersistenceProvider#isPinned(List) + */ public boolean[] isPinned(List transactionHashes) throws Exception { boolean[] mergedResult = new boolean[transactionHashes.size()]; for(PermanentPersistenceProvider provider: permanentPersistenceProviders) { diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index 3072fe526e..bf9e4b6864 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -3,7 +3,6 @@ import com.google.common.annotations.VisibleForTesting; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.model.HashFactory; import com.iota.iri.model.persistables.*; import com.iota.iri.model.persistables.Transaction; import com.iota.iri.storage.PermanentPersistenceProvider; @@ -24,23 +23,24 @@ import java.nio.file.Paths; import java.util.*; +/*** + * Implements the retrieval functions of a the normal persistence provider and the permanent persistence provider + */ public class RocksDBPPPImpl implements PermanentPersistenceProvider, PersistenceProvider { + private static final Logger log = LoggerFactory.getLogger(RocksDBPPPImpl.class); private static final int BLOOM_FILTER_BITS_PER_KEY = 10; - /**A delimeter for separating hashes within a byte stream*/ + /** A delimeter for separating hashes within a byte stream */ private static final byte delimiter = ",".getBytes()[0]; - private final List columnFamilyHandles = new ArrayList<>(); - - private final String dbPath; - private final String logPath; - private final int cacheSize; - public static String TRANSACTION_COLUMN = "transaction"; public static String ADDRESS_INDEX = "address-index"; public static String BUNDLE_INDEX = "bundle-index"; public static String TAG_INDEX = "tag-index"; public static String APPROVEE_INDEX = "approvee-index"; - + private final List columnFamilyHandles = new ArrayList<>(); + private final String dbPath; + private final String logPath; + private final int cacheSize; @VisibleForTesting public Map columnMap = new HashMap<>(); @@ -56,6 +56,10 @@ public RocksDBPPPImpl(String dbPath, String logPath, int cacheSize) { this.cacheSize = cacheSize; } + /** + * @see PersistenceProvider#init() + * @throws Exception + */ @Override public void init() throws Exception { log.info("Initializing Database on " + dbPath); @@ -64,11 +68,18 @@ public void init() throws Exception { log.info("RocksDB permanent persistence provider initialized."); } + /** + * @see PersistenceProvider#isAvailable() + * @return + */ @Override public boolean isAvailable() { return this.available; } + /** + * @see PersistenceProvider#shutdown() + */ @Override public void shutdown() { for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { @@ -77,48 +88,69 @@ public void shutdown() { IotaIOUtils.closeQuietly(db, options, bloomFilter); } - //----------- PersistenceProvider only overrides --------------- + // ----------- PersistenceProvider only overrides --------------- + /** + * @see PersistenceProvider#save(Persistable, Indexable) + */ @Override public boolean save(Persistable model, Indexable index) throws Exception { return false; } + /** + * @see PersistenceProvider#delete(Class, Indexable) + */ @Override public void delete(Class model, Indexable index) throws Exception { - //Do Nothing + // Do Nothing } + /** + * @see PersistenceProvider#update(Persistable, Indexable, String) + */ @Override public boolean update(Persistable model, Indexable index, String item) throws Exception { return false; } + /** + * @see PersistenceProvider#exists(Class, Indexable) () + */ @Override public boolean exists(Class model, Indexable key) throws Exception { return false; } + /** + * @see PersistenceProvider#latest(Class, Class) + */ @Override public Pair latest(Class model, Class indexModel) throws Exception { return null; } + /** + * @see PersistenceProvider#keysWithMissingReferences(Class, Class) + */ @Override public Set keysWithMissingReferences(Class modelClass, Class otherClass) throws Exception { return null; } + /** + * @see PersistenceProvider#get(Class, Indexable) + */ @Override public Persistable get(Class model, Indexable index) throws Exception { Persistable object = (Persistable) model.newInstance(); - if(object instanceof Transaction) { + if (object instanceof Transaction) { - TransactionViewModel tvm = getTransaction((Hash)index); - if(tvm == null){ + TransactionViewModel tvm = getTransaction((Hash) index); + if (tvm == null) { return object; } - //These 'manually' fill the metadata from the transaction data. + // These 'manually' fill the metadata from the transaction data. tvm.getAddressHash(); tvm.getBundleHash(); tvm.getBranchTransactionHash(); @@ -132,166 +164,225 @@ public Persistable get(Class model, Indexable index) throws Exception { return tvm.getTransaction(); } - if(object instanceof Bundle){ - Bundle toReturn = new Bundle(); - toReturn.set = findBundle((Hash)index).set; + if (object instanceof Bundle) { + Bundle toReturn = new Bundle(); + toReturn.set = findBundle((Hash) index).set; return toReturn; } - if(object instanceof Address){ - Address toReturn = new Address(); - toReturn.set = findAddress((Hash)index).set; + if (object instanceof Address) { + Address toReturn = new Address(); + toReturn.set = findAddress((Hash) index).set; return toReturn; } - if(object instanceof Tag){ - Tag toReturn = new Tag(); - toReturn.set = findTag((Hash)index).set; + if (object instanceof Tag) { + Tag toReturn = new Tag(); + toReturn.set = findTag((Hash) index).set; return toReturn; } - if(object instanceof Approvee){ - Approvee toReturn = new Approvee(); - toReturn.set = findApprovee((Hash)index).set; + if (object instanceof Approvee) { + Approvee toReturn = new Approvee(); + toReturn.set = findApprovee((Hash) index).set; return toReturn; } return object; } + /** + * @see PersistenceProvider#mayExist(Class, Indexable) + */ @Override public boolean mayExist(Class model, Indexable index) throws Exception { return false; } + /** + * @see PersistenceProvider#count(Class) + */ @Override public long count(Class model) throws Exception { return 0; } + /** + * @see PersistenceProvider#keysStartingWith(Class, byte[]) + */ @Override public Set keysStartingWith(Class modelClass, byte[] value) { return null; } + /** + * @see PersistenceProvider#seek(Class, byte[]) + */ @Override public Persistable seek(Class model, byte[] key) throws Exception { return null; } + /** + * @see PersistenceProvider#next(Class, Indexable) + */ @Override public Pair next(Class model, Indexable index) throws Exception { return null; } + /** + * @see PersistenceProvider#previous(Class, Indexable) + */ @Override public Pair previous(Class model, Indexable index) throws Exception { return null; } + /** + * @see PersistenceProvider#first(Class, Class) + */ @Override public Pair first(Class model, Class indexModel) throws Exception { return null; } + /** + * @see PersistenceProvider#saveBatch(List) + */ @Override public boolean saveBatch(List> models) throws Exception { return false; } + /** + * @see PersistenceProvider#deleteBatch(Collection) + */ @Override - public void deleteBatch(Collection>> models) throws Exception { + public void deleteBatch(Collection>> models) + throws Exception { } + /** + * @see PersistenceProvider#clear(Class) + */ @Override public void clear(Class column) throws Exception { } + /** + * @see PersistenceProvider#clearMetadata(Class) + */ @Override public void clearMetadata(Class column) throws Exception { } + /** + * @see PersistenceProvider#loadAllKeysFromTable(Class) + */ @Override public List loadAllKeysFromTable(Class model) { return null; } - //END----------- PersistenceProvider only overrides --------------- - + // END----------- PersistenceProvider only overrides --------------- + /** + * @see PermanentPersistenceProvider#pinTransaction(TransactionViewModel, Hash) + */ @Override public boolean pinTransaction(TransactionViewModel model, Hash index) throws Exception { boolean exists = isPinned(Collections.singletonList(index))[0]; - if(!exists){ - try (WriteBatch writeBatch = new WriteBatch(); - WriteOptions writeOptions = new WriteOptions()) { + if (!exists) { + try (WriteBatch writeBatch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions()) { byte[] txBytes = model.getBytes(); writeBatch.put(columnMap.get(TRANSACTION_COLUMN), index.bytes(), txBytes); - addToIndex(writeBatch, columnMap.get(ADDRESS_INDEX), model.getAddressHash(), index); - addToIndex(writeBatch, columnMap.get(TAG_INDEX), model.getTagValue(), index); - addToIndex(writeBatch, columnMap.get(BUNDLE_INDEX), model.getBundleHash(), index); - addToIndex(writeBatch, columnMap.get(APPROVEE_INDEX), model.getTrunkTransactionHash(), index); - addToIndex(writeBatch, columnMap.get(APPROVEE_INDEX), model.getBranchTransactionHash(), index); + addToIndex(writeBatch, columnMap.get(ADDRESS_INDEX), model.getAddressHash(), index); + addToIndex(writeBatch, columnMap.get(TAG_INDEX), model.getTagValue(), index); + addToIndex(writeBatch, columnMap.get(BUNDLE_INDEX), model.getBundleHash(), index); + addToIndex(writeBatch, columnMap.get(APPROVEE_INDEX), model.getTrunkTransactionHash(), index); + addToIndex(writeBatch, columnMap.get(APPROVEE_INDEX), model.getBranchTransactionHash(), index); db.write(writeOptions, writeBatch); return true; } - }else{ - return true; //Already pinned, is fine to return true + } else { + return true; // Already pinned, is fine to return true } } + /** + * @see PermanentPersistenceProvider#unpinTransaction(Hash) + */ @Override public boolean unpinTransaction(Hash index) throws Exception { - try (WriteBatch writeBatch = new WriteBatch(); - WriteOptions writeOptions = new WriteOptions()) { - + try (WriteBatch writeBatch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions()) { safeDeleteTransaction(writeBatch, index); db.write(writeOptions, writeBatch); return true; } } + /** + * @see PermanentPersistenceProvider#isPinned(List) + */ @Override public boolean[] isPinned(List indexes) throws Exception { boolean[] result = new boolean[indexes.size()]; ColumnFamilyHandle handle = columnMap.get(TRANSACTION_COLUMN); - for(int i = 0; i < result.length; i++){ + for (int i = 0; i < result.length; i++) { result[i] = db.keyMayExist(handle, indexes.get(i).bytes(), new StringBuilder()); } return result; } - + /** + * Atomic deletion of a transaction and their relevant indexes + * + * @param writeBatch rocksDB atomic write batch + * @param key transaction ID to delete + * @throws Exception + */ @VisibleForTesting public void safeDeleteTransaction(WriteBatch writeBatch, Hash key) throws Exception { byte[] keyBytes = key.bytes(); TransactionViewModel tx = getTransaction(key); writeBatch.delete(columnMap.get(TRANSACTION_COLUMN), keyBytes); - removeFromIndex(writeBatch, columnMap.get(ADDRESS_INDEX), tx.getAddressHash(), key); - removeFromIndex(writeBatch, columnMap.get(TAG_INDEX), tx.getTagValue(), key); - removeFromIndex(writeBatch, columnMap.get(BUNDLE_INDEX), tx.getBundleHash(), key); - removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getTrunkTransactionHash(), key); - removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getBranchTransactionHash(), key); + removeFromIndex(writeBatch, columnMap.get(ADDRESS_INDEX), tx.getAddressHash(), key); + removeFromIndex(writeBatch, columnMap.get(TAG_INDEX), tx.getTagValue(), key); + removeFromIndex(writeBatch, columnMap.get(BUNDLE_INDEX), tx.getBundleHash(), key); + removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getTrunkTransactionHash(), key); + removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getBranchTransactionHash(), key); } + + /** + * Removes an index from a set and deletes the index when the set is empty. + * With an atomic write batch + * @param writeBatch rocksDB atomic operation + * @param column index column + * @param key index key + * @param indexValue index value + * @throws Exception + */ @VisibleForTesting - public void removeFromIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) throws Exception{ + public void removeFromIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) + throws Exception { byte[] indexBytes = indexValue.bytes(); byte[] result = db.get(column, key.bytes()); - if(result != null) { - if(result.length <= indexBytes.length){ - //when it is the last one to delete just delete the entire index key space. + if (result != null) { + if (result.length <= indexBytes.length) { + // when it is the last one to delete just delete the entire index key space. writeBatch.delete(column, key.bytes()); - }else{ + } else { int keyLoc = indexOf(result, indexBytes); - if(keyLoc >= 0){ //Does exists - ByteBuffer buffer = ByteBuffer.allocate( result.length - indexBytes.length - 1); + if (keyLoc >= 0) { // Does exists + ByteBuffer buffer = ByteBuffer.allocate(result.length - indexBytes.length - 1); byte[] subarray1 = ArrayUtils.subarray(result, 0, keyLoc - 1); buffer.put(subarray1); byte[] subarray2 = ArrayUtils.subarray(result, keyLoc + indexBytes.length + 1, result.length); - if(subarray1.length > 0 && subarray2.length > 0){ + if (subarray1.length > 0 && subarray2.length > 0) { buffer.put(delimiter); buffer.put(subarray2); } @@ -301,22 +392,33 @@ public void removeFromIndex(WriteBatch writeBatch, ColumnFamilyHandle column, In } } + /** + * Adds a key to an index through an atomic write batch + * @param writeBatch rocksDB atomic operation + * @param column index column + * @param key index key + * @param indexValue index value + * @throws Exception + */ @VisibleForTesting - public void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) throws Exception{ + public void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) + throws Exception { byte[] indexBytes = indexValue.bytes(); byte[] dbResult = db.get(column, key.bytes()); - if(dbResult == null) { + if (dbResult == null) { dbResult = new byte[0]; } int keyLoc = indexOf(dbResult, indexBytes); - if(keyLoc == -1){//not found + if (keyLoc == -1) {// not found - ByteBuffer buffer = ByteBuffer.allocate(dbResult.length + indexBytes.length + (dbResult.length > 0 ? 1 : 0));//+1 delimiter length + ByteBuffer buffer = ByteBuffer + .allocate(dbResult.length + indexBytes.length + (dbResult.length > 0 ? 1 : 0));// +1 delimiter + // length buffer.put(dbResult); - if(dbResult.length > 0){ - //Means we add on the end of the current stream. - //To be compatible with Hashes.read(byte[]) + if (dbResult.length > 0) { + // Means we add on the end of the current stream. + // To be compatible with Hashes.read(byte[]) buffer.put(delimiter); } buffer.put(indexBytes); @@ -324,57 +426,79 @@ public void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexab } } - + /** + * Byte matching variant of string.indexOf + * @param outerArray Array to search in + * @param smallerArray What to find + * @return + */ public int indexOf(byte[] outerArray, byte[] smallerArray) { - for(int i = 0; i < outerArray.length - smallerArray.length+1; ++i) { + for (int i = 0; i < outerArray.length - smallerArray.length + 1; ++i) { boolean found = true; - for(int j = 0; j < smallerArray.length; ++j) { - if (outerArray[i+j] != smallerArray[j]) { + for (int j = 0; j < smallerArray.length; ++j) { + if (outerArray[i + j] != smallerArray[j]) { found = false; break; } } - if (found) {return i;} + if (found) { + return i; + } } return -1; } - + /** + * @see PermanentPersistenceProvider#getTransaction(Hash) + */ @Override public TransactionViewModel getTransaction(Hash index) throws Exception { - if(index == null){return null;} + if (index == null) { + return null; + } byte[] dbResult = db.get(columnMap.get(TRANSACTION_COLUMN), index.bytes()); - if(dbResult != null && dbResult.length > 0){ + if (dbResult != null && dbResult.length > 0) { return new TransactionViewModel(TransactionViewModel.trits(dbResult), Hash.NULL_HASH); } return null; } - + /** + * @see PermanentPersistenceProvider#findAddress(Hash) + */ @Override public Hashes findAddress(Hash index) throws Exception { - return getIndex(columnMap.get(ADDRESS_INDEX), index); + return getIndex(columnMap.get(ADDRESS_INDEX), index); } + /** + * @see PermanentPersistenceProvider#findBundle(Hash) + */ @Override public Hashes findBundle(Hash index) throws Exception { return getIndex(columnMap.get(BUNDLE_INDEX), index); } + /** + * @see PermanentPersistenceProvider#findTag(Hash) + */ @Override public Hashes findTag(Hash index) throws Exception { return getIndex(columnMap.get(TAG_INDEX), index); } + /** + * @see PermanentPersistenceProvider#findApprovee(Hash) + */ @Override public Hashes findApprovee(Hash index) throws Exception { return getIndex(columnMap.get(APPROVEE_INDEX), index); } - private Hashes getIndex(ColumnFamilyHandle column, Indexable index) throws Exception { - if(index == null){return null;} + if (index == null) { + return null; + } Hashes found = new Hashes(); byte[] result = db.get(column, index.bytes()); - if(result != null) { + if (result != null) { found.read(result); - } return found; } @@ -385,9 +509,8 @@ private void initDB(String path, String logPath) throws Exception { RocksDB.loadLibrary(); } catch (Exception e) { if (SystemUtils.IS_OS_WINDOWS) { - log.error("Error loading RocksDB library. Please ensure that " + - "Microsoft Visual C++ 2015 Redistributable Update 3 " + - "is installed and updated"); + log.error("Error loading RocksDB library. Please ensure that " + + "Microsoft Visual C++ 2015 Redistributable Update 3 " + "is installed and updated"); } throw e; } @@ -401,17 +524,11 @@ private void initDB(String path, String logPath) throws Exception { } int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); - RocksEnv.getDefault() - .setBackgroundThreads(numThreads, RocksEnv.FLUSH_POOL) - .setBackgroundThreads(numThreads, RocksEnv.COMPACTION_POOL); - - options = new DBOptions() - .setCreateIfMissing(true) - .setCreateMissingColumnFamilies(true) - .setDbLogDir(logPath) - .setMaxLogFileSize(SizeUnit.MB) - .setMaxManifestFileSize(SizeUnit.MB) - .setMaxOpenFiles(10000) + RocksEnv.getDefault().setBackgroundThreads(numThreads, RocksEnv.FLUSH_POOL).setBackgroundThreads(numThreads, + RocksEnv.COMPACTION_POOL); + + options = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true).setDbLogDir(logPath) + .setMaxLogFileSize(SizeUnit.MB).setMaxManifestFileSize(SizeUnit.MB).setMaxOpenFiles(10000) .setMaxBackgroundCompactions(1); options.setMaxSubcompactions(Runtime.getRuntime().availableProcessors()); @@ -419,49 +536,44 @@ private void initDB(String path, String logPath) throws Exception { bloomFilter = new BloomFilter(BLOOM_FILTER_BITS_PER_KEY); BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig().setFilter(bloomFilter); - blockBasedTableConfig - .setFilter(bloomFilter) - .setCacheNumShardBits(2) - .setBlockSizeDeviation(10) - .setBlockRestartInterval(16) - .setBlockCacheSize(cacheSize * SizeUnit.KB) - .setBlockCacheCompressedNumShardBits(10) - .setBlockCacheCompressedSize(32 * SizeUnit.KB); + blockBasedTableConfig.setFilter(bloomFilter).setCacheNumShardBits(2).setBlockSizeDeviation(10) + .setBlockRestartInterval(16).setBlockCacheSize(cacheSize * SizeUnit.KB) + .setBlockCacheCompressedNumShardBits(10).setBlockCacheCompressedSize(32 * SizeUnit.KB); options.setAllowConcurrentMemtableWrite(true); MergeOperator mergeOperator = new StringAppendOperator(); - ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions() - .setMergeOperator(mergeOperator) - .setTableFormatConfig(blockBasedTableConfig) - .setMaxWriteBufferNumber(2) + ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions().setMergeOperator(mergeOperator) + .setTableFormatConfig(blockBasedTableConfig).setMaxWriteBufferNumber(2) .setWriteBufferSize(2 * SizeUnit.MB); List columnFamilyDescriptors = new ArrayList<>(); - //Add default column family. Main motivation is to not change legacy code + // Add default column family. Main motivation is to not change legacy code columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions)); - columnFamilyDescriptors.add(new ColumnFamilyDescriptor(TRANSACTION_COLUMN.getBytes("UTF-8"), columnFamilyOptions)); - // columnFamilyDescriptors.add(new ColumnFamilyDescriptor(COUNTER_INDEX.getBytes("UTF-8"), columnFamilyOptions)); - columnFamilyDescriptors.add(new ColumnFamilyDescriptor(BUNDLE_INDEX.getBytes("UTF-8"), columnFamilyOptions)); + columnFamilyDescriptors + .add(new ColumnFamilyDescriptor(TRANSACTION_COLUMN.getBytes("UTF-8"), columnFamilyOptions)); + // columnFamilyDescriptors.add(new ColumnFamilyDescriptor(COUNTER_INDEX.getBytes("UTF-8"), + // columnFamilyOptions)); + columnFamilyDescriptors + .add(new ColumnFamilyDescriptor(BUNDLE_INDEX.getBytes("UTF-8"), columnFamilyOptions)); columnFamilyDescriptors.add(new ColumnFamilyDescriptor(TAG_INDEX.getBytes("UTF-8"), columnFamilyOptions)); - columnFamilyDescriptors.add(new ColumnFamilyDescriptor(ADDRESS_INDEX.getBytes("UTF-8"), columnFamilyOptions)); - columnFamilyDescriptors.add(new ColumnFamilyDescriptor(APPROVEE_INDEX.getBytes("UTF-8"), columnFamilyOptions)); + columnFamilyDescriptors + .add(new ColumnFamilyDescriptor(ADDRESS_INDEX.getBytes("UTF-8"), columnFamilyOptions)); + columnFamilyDescriptors + .add(new ColumnFamilyDescriptor(APPROVEE_INDEX.getBytes("UTF-8"), columnFamilyOptions)); db = RocksDB.open(options, path, columnFamilyDescriptors, columnFamilyHandles); db.enableFileDeletions(true); - for (ColumnFamilyHandle columnHandler: columnFamilyHandles) { - columnMap.put(new String(columnHandler.getName(), "UTF-8" ), columnHandler); + for (ColumnFamilyHandle columnHandler : columnFamilyHandles) { + columnMap.put(new String(columnHandler.getName(), "UTF-8"), columnHandler); } - } catch (Exception e) { IotaIOUtils.closeQuietly(db); throw e; } } - - } diff --git a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java index 33b1124fed..e4688f45e4 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/PermaRocksDBPersentenceProviderTest.java @@ -3,9 +3,11 @@ import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.storage.Indexable; import org.apache.commons.io.FileUtils; -import org.junit.*; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; import java.io.File; import java.util.Collections; From 1416adacf48cab526aa3841af87ec5c486ead97f Mon Sep 17 00:00:00 2001 From: olaf Date: Tue, 1 Oct 2019 11:37:16 +0200 Subject: [PATCH 13/26] Style updates --- src/main/java/com/iota/iri/Iota.java | 2 -- src/main/java/com/iota/iri/conf/PermaDBConfig.java | 3 +++ .../iri/service/dto/BooleanValuesResponse.java | 14 ++++++++++++-- .../iri/storage/PermanentPersistenceProvider.java | 3 +++ src/main/java/com/iota/iri/storage/Tangle.java | 9 +++++++++ .../iota/iri/storage/rocksDB/RocksDBPPPImpl.java | 10 +++++----- 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index 40b2684746..5904e9c835 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -1,7 +1,5 @@ package com.iota.iri; -import java.security.SecureRandom; -import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/src/main/java/com/iota/iri/conf/PermaDBConfig.java b/src/main/java/com/iota/iri/conf/PermaDBConfig.java index 01a623fae8..dea784c055 100644 --- a/src/main/java/com/iota/iri/conf/PermaDBConfig.java +++ b/src/main/java/com/iota/iri/conf/PermaDBConfig.java @@ -55,6 +55,9 @@ public interface PermaDBConfig extends Config { */ boolean permaIsRescanDb(); + /** + * Field descriptions + */ interface Descriptions { String PERMADB_PATH = "The folder where the DB saves its data."; diff --git a/src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java b/src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java index 0764f6a8db..dd5c794568 100644 --- a/src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java +++ b/src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java @@ -1,19 +1,29 @@ package com.iota.iri.service.dto; + + public class BooleanValuesResponse extends AbstractResponse { /** - * States of the specified addresses in Boolean - * Order of booleans is equal to order of the supplied addresses. + * List of boleans to use as an API result */ private boolean[] result; + /** + * Creates a new {@link BooleanValuesResponse} + * @param values {@link #result} + * @return an {@link BooleanValuesResponse} filled with a list of booleans + */ public static AbstractResponse create(boolean[] values) { BooleanValuesResponse res = new BooleanValuesResponse(); res.result = values; return res; } + /** + * + * @return list of booleans + */ public boolean[] getResult() { return this.result; } diff --git a/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java b/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java index c1da6793ac..d6088a2a0b 100644 --- a/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/PermanentPersistenceProvider.java @@ -6,6 +6,9 @@ import java.util.List; +/** + * Abstracts the permanent storage for transaction pinning + */ public interface PermanentPersistenceProvider { diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index f01d1d49c1..340ed0d31f 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -46,9 +46,18 @@ public class Tangle { private final List permanentPersistenceProviders = new ArrayList<>(); private final List messageQueueProviders = new ArrayList<>(); + /** + * Adds a persistence provider + * @param provider + */ public void addPersistenceProvider(PersistenceProvider provider) { this.persistenceProviders.add(provider); } + + /** + * Adds a perment persistence provider + * @param provider + */ public void addPermanentPersistenceProvider(PermanentPersistenceProvider provider) { this.permanentPersistenceProviders.add(provider); } diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index bf9e4b6864..e5677fb9b8 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -13,6 +13,7 @@ import com.iota.iri.utils.Pair; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.SystemUtils; + import org.rocksdb.*; import org.rocksdb.util.SizeUnit; import org.slf4j.Logger; @@ -102,7 +103,7 @@ public boolean save(Persistable model, Indexable index) throws Exception { */ @Override public void delete(Class model, Indexable index) throws Exception { - // Do Nothing + return; } /** @@ -257,7 +258,7 @@ public boolean saveBatch(List> models) throws Excep @Override public void deleteBatch(Collection>> models) throws Exception { - + return; } /** @@ -265,7 +266,7 @@ public void deleteBatch(Collection column) throws Exception { - + return; } /** @@ -273,7 +274,7 @@ public void clear(Class column) throws Exception { */ @Override public void clearMetadata(Class column) throws Exception { - + return; } /** @@ -354,7 +355,6 @@ public void safeDeleteTransaction(WriteBatch writeBatch, Hash key) throws Except removeFromIndex(writeBatch, columnMap.get(BUNDLE_INDEX), tx.getBundleHash(), key); removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getTrunkTransactionHash(), key); removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getBranchTransactionHash(), key); - } /** From 68680f8067cdeb810a0cd6363123f8be2a81978a Mon Sep 17 00:00:00 2001 From: olaf Date: Tue, 1 Oct 2019 11:51:10 +0200 Subject: [PATCH 14/26] Import hell, why is this better then .* ? --- .../iri/storage/rocksDB/RocksDBPPPImpl.java | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index e5677fb9b8..c0049bafea 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -3,8 +3,12 @@ import com.google.common.annotations.VisibleForTesting; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; -import com.iota.iri.model.persistables.*; import com.iota.iri.model.persistables.Transaction; +import com.iota.iri.model.persistables.Tag; +import com.iota.iri.model.persistables.Bundle; +import com.iota.iri.model.persistables.Address; +import com.iota.iri.model.persistables.Approvee; +import com.iota.iri.model.persistables.Hashes; import com.iota.iri.storage.PermanentPersistenceProvider; import com.iota.iri.storage.Indexable; import com.iota.iri.storage.Persistable; @@ -14,15 +18,33 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.SystemUtils; -import org.rocksdb.*; import org.rocksdb.util.SizeUnit; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.RocksDB; +import org.rocksdb.DBOptions; +import org.rocksdb.BloomFilter; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; +import org.rocksdb.RocksEnv; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.StringAppendOperator; +import org.rocksdb.MergeOperator; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.ColumnFamilyDescriptor; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.nio.ByteBuffer; import java.nio.file.Paths; -import java.util.*; +import java.util.Set; +import java.util.Map; +import java.util.HashMap; +import java.util.List; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; /*** * Implements the retrieval functions of a the normal persistence provider and the permanent persistence provider From 74f86e7cdc6ddb7ba2dc1ad0fc2209499bd848df Mon Sep 17 00:00:00 2001 From: olaf Date: Tue, 1 Oct 2019 11:55:10 +0200 Subject: [PATCH 15/26] Constructor comment and boolean response comment --- .../com/iota/iri/service/dto/BooleanValuesResponse.java | 4 +++- .../java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java b/src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java index dd5c794568..c3f29956f3 100644 --- a/src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java +++ b/src/main/java/com/iota/iri/service/dto/BooleanValuesResponse.java @@ -1,7 +1,9 @@ package com.iota.iri.service.dto; - +/** + * Genereric boolean list response + */ public class BooleanValuesResponse extends AbstractResponse { /** * List of boleans to use as an API result diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index c0049bafea..251a40fb44 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -73,6 +73,12 @@ public class RocksDBPPPImpl implements PermanentPersistenceProvider, Persistence private BloomFilter bloomFilter; private boolean available; + /** + * Constructor + * @param dbPath database path + * @param logPath database log path + * @param cacheSize cache size + */ public RocksDBPPPImpl(String dbPath, String logPath, int cacheSize) { this.dbPath = dbPath; this.logPath = logPath; From 0c96323c9657096a2185c36221787f4616bb6744 Mon Sep 17 00:00:00 2001 From: Olaf van Wijk Date: Sun, 10 Nov 2019 17:16:38 +0100 Subject: [PATCH 16/26] Update src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java Co-Authored-By: Gal Rogozinski --- src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index 251a40fb44..510ca50d05 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -65,7 +65,7 @@ public class RocksDBPPPImpl implements PermanentPersistenceProvider, Persistence private final String logPath; private final int cacheSize; @VisibleForTesting - public Map columnMap = new HashMap<>(); + Map columnMap = new HashMap<>(); private RocksDB db; // DBOptions is only used in initDB(). However, it is closeable - so we keep a reference for shutdown. From 734fa0420ae1f5b1e28d42d713bae453197a476e Mon Sep 17 00:00:00 2001 From: Olaf van Wijk Date: Sun, 10 Nov 2019 17:16:46 +0100 Subject: [PATCH 17/26] Update src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java Co-Authored-By: Gal Rogozinski --- src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index 510ca50d05..6c0f2de9a1 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -373,7 +373,7 @@ public boolean[] isPinned(List indexes) throws Exception { * @throws Exception */ @VisibleForTesting - public void safeDeleteTransaction(WriteBatch writeBatch, Hash key) throws Exception { + void safeDeleteTransaction(WriteBatch writeBatch, Hash key) throws Exception { byte[] keyBytes = key.bytes(); TransactionViewModel tx = getTransaction(key); writeBatch.delete(columnMap.get(TRANSACTION_COLUMN), keyBytes); From 5d467d9905753ef997ece7265a68452ef79b61e7 Mon Sep 17 00:00:00 2001 From: Olaf van Wijk Date: Sun, 10 Nov 2019 17:16:52 +0100 Subject: [PATCH 18/26] Update src/main/java/com/iota/iri/service/API.java Co-Authored-By: Gal Rogozinski --- src/main/java/com/iota/iri/service/API.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index ae8b9b5dfe..be52c4c70e 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -1838,7 +1838,7 @@ private Function, AbstractResponse> isPinned() { private Function, AbstractResponse> unpinTransactionHashes() { return request -> { - final List txids = getParameterAsList(request,"hashes", HASH_SIZE); + List txids = getParameterAsList(request,"hashes", HASH_SIZE); try { return unpinTransactionHashesStatement(txids); } catch (Exception e) { From 9dee4cd7d9d493d572deef73450b88d0516b7b57 Mon Sep 17 00:00:00 2001 From: Olaf van Wijk Date: Sun, 10 Nov 2019 17:17:03 +0100 Subject: [PATCH 19/26] Update src/main/java/com/iota/iri/service/API.java Co-Authored-By: Gal Rogozinski --- src/main/java/com/iota/iri/service/API.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index be52c4c70e..2639cfee50 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -1805,7 +1805,7 @@ private List convertTrytes(List trytes) { private Function, AbstractResponse> pinTransactionTrytes() { return request -> { - final List transactionTrytes = getParameterAsList(request,"trytes", TRYTES_SIZE); + List transactionTrytes = getParameterAsList(request,"trytes", TRYTES_SIZE); try { return pinTransactionTrytesStatement(transactionTrytes); } catch (Exception e) { From e6288e950745646905237f33faa8afa4eddc5b12 Mon Sep 17 00:00:00 2001 From: Olaf van Wijk Date: Sun, 10 Nov 2019 17:17:10 +0100 Subject: [PATCH 20/26] Update src/main/java/com/iota/iri/service/API.java Co-Authored-By: Gal Rogozinski --- src/main/java/com/iota/iri/service/API.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 2639cfee50..04275fdf6b 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -1827,7 +1827,7 @@ private Function, AbstractResponse> pinTransactionHashes() { private Function, AbstractResponse> isPinned() { return request -> { - final List txids = getParameterAsList(request,"hashes", HASH_SIZE); + List txids = getParameterAsList(request,"hashes", HASH_SIZE); try { return isPinnedStatement(txids); } catch (Exception e) { From 06d35b8f31595a3a036f7b08c94ee32fce858037 Mon Sep 17 00:00:00 2001 From: Olaf van Wijk Date: Sun, 10 Nov 2019 17:17:17 +0100 Subject: [PATCH 21/26] Update src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java Co-Authored-By: Gal Rogozinski --- src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index 6c0f2de9a1..2a4e0527b9 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -429,7 +429,7 @@ public void removeFromIndex(WriteBatch writeBatch, ColumnFamilyHandle column, In * @throws Exception */ @VisibleForTesting - public void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) + void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) throws Exception { byte[] indexBytes = indexValue.bytes(); byte[] dbResult = db.get(column, key.bytes()); From bf0fb207edffdf00fd47d93f38d8fb67534e698c Mon Sep 17 00:00:00 2001 From: Olaf van Wijk Date: Sun, 10 Nov 2019 17:17:26 +0100 Subject: [PATCH 22/26] Update src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java Co-Authored-By: Gal Rogozinski --- src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index 2a4e0527b9..366e2f506c 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -395,7 +395,7 @@ void safeDeleteTransaction(WriteBatch writeBatch, Hash key) throws Exception { * @throws Exception */ @VisibleForTesting - public void removeFromIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) + void removeFromIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) throws Exception { byte[] indexBytes = indexValue.bytes(); byte[] result = db.get(column, key.bytes()); From 7faad3850a69dd242d9be8dbd60f2a812a400134 Mon Sep 17 00:00:00 2001 From: Olaf van Wijk Date: Sun, 10 Nov 2019 17:17:37 +0100 Subject: [PATCH 23/26] Update src/main/java/com/iota/iri/service/API.java Co-Authored-By: Gal Rogozinski --- src/main/java/com/iota/iri/service/API.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/service/API.java b/src/main/java/com/iota/iri/service/API.java index 04275fdf6b..d4547b751a 100644 --- a/src/main/java/com/iota/iri/service/API.java +++ b/src/main/java/com/iota/iri/service/API.java @@ -1816,7 +1816,7 @@ private Function, AbstractResponse> pinTransactionTrytes() { private Function, AbstractResponse> pinTransactionHashes() { return request -> { - final List txids = getParameterAsList(request,"hashes", HASH_SIZE); + List txids = getParameterAsList(request,"hashes", HASH_SIZE); try { return pinTransactionHashesStatement(txids); } catch (Exception e) { From 5abc785f1634da922d4f2ab9a59bf54920897c9a Mon Sep 17 00:00:00 2001 From: olaf Date: Tue, 19 Nov 2019 10:04:31 +0100 Subject: [PATCH 24/26] Commit and merge --- .../java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index 366e2f506c..ecc19aab1d 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -375,6 +375,7 @@ public boolean[] isPinned(List indexes) throws Exception { @VisibleForTesting void safeDeleteTransaction(WriteBatch writeBatch, Hash key) throws Exception { byte[] keyBytes = key.bytes(); + //TODO Handle null TransactionViewModel tx = getTransaction(key); writeBatch.delete(columnMap.get(TRANSACTION_COLUMN), keyBytes); @@ -552,8 +553,8 @@ private void initDB(String path, String logPath) throws Exception { } int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); - RocksEnv.getDefault().setBackgroundThreads(numThreads, RocksEnv.FLUSH_POOL).setBackgroundThreads(numThreads, - RocksEnv.COMPACTION_POOL); + RocksEnv.getDefault().setBackgroundThreads(numThreads, Priority.HIGH).setBackgroundThreads(numThreads, + Priority.LOW); options = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true).setDbLogDir(logPath) .setMaxLogFileSize(SizeUnit.MB).setMaxManifestFileSize(SizeUnit.MB).setMaxOpenFiles(10000) From e5a8f1619073504264be7cfa406e10831710536f Mon Sep 17 00:00:00 2001 From: olaf Date: Tue, 19 Nov 2019 15:10:20 +0100 Subject: [PATCH 25/26] Null fix and mayExist fix --- .../iri/storage/rocksDB/RocksDBPPPImpl.java | 64 ++++++++----------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java index ecc19aab1d..4de8f0d05e 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPPPImpl.java @@ -1,6 +1,7 @@ package com.iota.iri.storage.rocksDB; import com.google.common.annotations.VisibleForTesting; +import com.google.common.primitives.Bytes; import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.model.persistables.Transaction; @@ -21,6 +22,7 @@ import org.rocksdb.util.SizeUnit; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDB; +import org.rocksdb.Priority; import org.rocksdb.DBOptions; import org.rocksdb.BloomFilter; import org.rocksdb.WriteBatch; @@ -349,6 +351,9 @@ public boolean unpinTransaction(Hash index) throws Exception { safeDeleteTransaction(writeBatch, index); db.write(writeOptions, writeBatch); return true; + }catch (Exception e) { + log.error(e.getLocalizedMessage()); + return false; } } @@ -360,7 +365,13 @@ public boolean[] isPinned(List indexes) throws Exception { boolean[] result = new boolean[indexes.size()]; ColumnFamilyHandle handle = columnMap.get(TRANSACTION_COLUMN); for (int i = 0; i < result.length; i++) { - result[i] = db.keyMayExist(handle, indexes.get(i).bytes(), new StringBuilder()); + byte[] keyBytes = indexes.get(i).bytes(); + if(db.keyMayExist(handle, keyBytes, new StringBuilder())){ + result[i] = db.get(handle, keyBytes) != null; + }else{ + result[i] = false; + } + } return result; } @@ -375,15 +386,16 @@ public boolean[] isPinned(List indexes) throws Exception { @VisibleForTesting void safeDeleteTransaction(WriteBatch writeBatch, Hash key) throws Exception { byte[] keyBytes = key.bytes(); - //TODO Handle null TransactionViewModel tx = getTransaction(key); - writeBatch.delete(columnMap.get(TRANSACTION_COLUMN), keyBytes); - - removeFromIndex(writeBatch, columnMap.get(ADDRESS_INDEX), tx.getAddressHash(), key); - removeFromIndex(writeBatch, columnMap.get(TAG_INDEX), tx.getTagValue(), key); - removeFromIndex(writeBatch, columnMap.get(BUNDLE_INDEX), tx.getBundleHash(), key); - removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getTrunkTransactionHash(), key); - removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getBranchTransactionHash(), key); + if(tx != null) { + writeBatch.delete(columnMap.get(TRANSACTION_COLUMN), keyBytes); + + removeFromIndex(writeBatch, columnMap.get(ADDRESS_INDEX), tx.getAddressHash(), key); + removeFromIndex(writeBatch, columnMap.get(TAG_INDEX), tx.getTagValue(), key); + removeFromIndex(writeBatch, columnMap.get(BUNDLE_INDEX), tx.getBundleHash(), key); + removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getTrunkTransactionHash(), key); + removeFromIndex(writeBatch, columnMap.get(APPROVEE_INDEX), tx.getBranchTransactionHash(), key); + } } /** @@ -405,7 +417,8 @@ void removeFromIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable // when it is the last one to delete just delete the entire index key space. writeBatch.delete(column, key.bytes()); } else { - int keyLoc = indexOf(result, indexBytes); + + int keyLoc = Bytes.indexOf(result, indexBytes); if (keyLoc >= 0) { // Does exists ByteBuffer buffer = ByteBuffer.allocate(result.length - indexBytes.length - 1); byte[] subarray1 = ArrayUtils.subarray(result, 0, keyLoc - 1); @@ -433,13 +446,10 @@ void removeFromIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, Indexable indexValue) throws Exception { byte[] indexBytes = indexValue.bytes(); + byte[] dbResult = db.get(column, key.bytes()); - if (dbResult == null) { - dbResult = new byte[0]; - } - int keyLoc = indexOf(dbResult, indexBytes); - if (keyLoc == -1) {// not found + if (dbResult != null && Bytes.indexOf(dbResult, indexBytes) != -1) {// not found ByteBuffer buffer = ByteBuffer .allocate(dbResult.length + indexBytes.length + (dbResult.length > 0 ? 1 : 0));// +1 delimiter @@ -455,28 +465,6 @@ void addToIndex(WriteBatch writeBatch, ColumnFamilyHandle column, Indexable key, } } - /** - * Byte matching variant of string.indexOf - * @param outerArray Array to search in - * @param smallerArray What to find - * @return - */ - public int indexOf(byte[] outerArray, byte[] smallerArray) { - for (int i = 0; i < outerArray.length - smallerArray.length + 1; ++i) { - boolean found = true; - for (int j = 0; j < smallerArray.length; ++j) { - if (outerArray[i + j] != smallerArray[j]) { - found = false; - break; - } - } - if (found) { - return i; - } - } - return -1; - } - /** * @see PermanentPersistenceProvider#getTransaction(Hash) */ @@ -522,7 +510,7 @@ public Hashes findApprovee(Hash index) throws Exception { private Hashes getIndex(ColumnFamilyHandle column, Indexable index) throws Exception { if (index == null) { - return null; + return new Hashes(); } Hashes found = new Hashes(); byte[] result = db.get(column, index.bytes()); From 97d5ec0e16ac2aba7a47922871a7e978eb806aac Mon Sep 17 00:00:00 2001 From: olaf Date: Mon, 20 Jan 2020 11:32:24 +0100 Subject: [PATCH 26/26] Fix API check --- src/main/java/com/iota/iri/storage/Tangle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iota/iri/storage/Tangle.java b/src/main/java/com/iota/iri/storage/Tangle.java index 340ed0d31f..e20ab6d2f2 100644 --- a/src/main/java/com/iota/iri/storage/Tangle.java +++ b/src/main/java/com/iota/iri/storage/Tangle.java @@ -403,7 +403,7 @@ public boolean pinTransaction(TransactionViewModel tvm) throws Exception { */ public boolean pinTransaction(Hash hash) throws Exception { Transaction tx = (Transaction)load(Transaction.class, hash); - if(!tx.exists()) { + if(tx.exists()) { TransactionViewModel tvm = new TransactionViewModel(tx, hash); return pinTransaction(tvm); }