From 454c471dfef7eef0d97604e82358dcfefad1bf7b Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 3 Sep 2024 18:39:41 -0700 Subject: [PATCH 1/3] Changed gapLimit from 24 to 3 since we have mitigated the gap problem. --- src/main/java/org/qortal/settings/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index de0ce5eda..90dab19b2 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -323,7 +323,7 @@ public class Settings { /* Foreign chains */ /** The number of consecutive empty addresses required before treating a wallet's transaction set as complete */ - private int gapLimit = 24; + private int gapLimit = 3; /** How many wallet keys to generate when using bitcoinj as the blockchain interface (e.g. when sending coins) */ private int bitcoinjLookaheadSize = 50; From acc37cef0e90536c319842456fc9cecbc28d5659 Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 3 Sep 2024 18:42:27 -0700 Subject: [PATCH 2/3] storing blockchain data in a cache to reduce redundant RPCs to the ElectrumX servers --- .../java/org/qortal/crosschain/Bitcoiny.java | 68 ++++++++++---- .../qortal/crosschain/BlockchainCache.java | 91 +++++++++++++++++++ 2 files changed, 143 insertions(+), 16 deletions(-) create mode 100644 src/main/java/org/qortal/crosschain/BlockchainCache.java diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 4a8192097..a4f5a2af0 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -55,6 +55,13 @@ public abstract class Bitcoiny implements ForeignBlockchain { protected Coin feePerKb; + /** + * Blockchain Cache + * + * To store blockchain data and reduce redundant RPCs to the ElectrumX servers + */ + private final BlockchainCache blockchainCache = new BlockchainCache(); + // Constructors and instance protected Bitcoiny(BitcoinyBlockchainProvider blockchainProvider, Context bitcoinjContext, String currencyCode, Coin feePerKb) { @@ -509,8 +516,22 @@ public List getWalletTransactions(String key58) throws Foreig if (!historicTransactionHashes.isEmpty()) { areAllKeysUnused = false; - for (TransactionHash transactionHash : historicTransactionHashes) - walletTransactions.add(this.getTransaction(transactionHash.txHash)); + for (TransactionHash transactionHash : historicTransactionHashes) { + + Optional walletTransaction + = this.blockchainCache.getTransactionByHash( transactionHash.txHash ); + + // if the wallet transaction is already cached + if(walletTransaction.isPresent() ) { + walletTransactions.add( walletTransaction.get() ); + } + // otherwise get the transaction from the blockchain server + else { + BitcoinyTransaction transaction = getTransaction(transactionHash.txHash); + walletTransactions.add( transaction ); + this.blockchainCache.addTransactionByHash(transactionHash.txHash, transaction); + } + } } } @@ -602,17 +623,25 @@ public Set getWalletAddresses(String key58) throws ForeignBlockchainExce for (; ki < keys.size(); ++ki) { DeterministicKey dKey = keys.get(ki); - // Check for transactions Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); keySet.add(address.toString()); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - - // Ask for transaction history - if it's empty then key has never been used - List historicTransactionHashes = this.getAddressTransactions(script, true); - if (!historicTransactionHashes.isEmpty()) { + // if the key already has a verified transaction history + if( this.blockchainCache.keyHasHistory( dKey ) ){ areAllKeysUnused = false; } + else { + // Check for transactions + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes = this.getAddressTransactions(script, true); + + if (!historicTransactionHashes.isEmpty()) { + areAllKeysUnused = false; + this.blockchainCache.addKeyWithHistory(dKey); + } + } } if (areAllKeysUnused) { @@ -667,18 +696,25 @@ private List getOldWalletKeys(String masterPrivateKey) throws do { boolean areAllKeysUnused = true; - for (; ki < keys.size(); ++ki) { + for (; areAllKeysUnused && ki < keys.size(); ++ki) { DeterministicKey dKey = keys.get(ki); - // Check for transactions - Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + // if the key already has a verified transaction history + if( this.blockchainCache.keyHasHistory(dKey)) { + areAllKeysUnused = false; + } + else { + // Check for transactions + Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - // Ask for transaction history - if it's empty then key has never been used - List historicTransactionHashes = this.getAddressTransactions(script, false); + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes = this.getAddressTransactions(script, true); - if (!historicTransactionHashes.isEmpty()) { - areAllKeysUnused = false; + if (!historicTransactionHashes.isEmpty()) { + areAllKeysUnused = false; + this.blockchainCache.addKeyWithHistory(dKey); + } } } diff --git a/src/main/java/org/qortal/crosschain/BlockchainCache.java b/src/main/java/org/qortal/crosschain/BlockchainCache.java new file mode 100644 index 000000000..bfc6f3c6c --- /dev/null +++ b/src/main/java/org/qortal/crosschain/BlockchainCache.java @@ -0,0 +1,91 @@ +package org.qortal.crosschain; + +import org.bitcoinj.crypto.DeterministicKey; + +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; + +/** + * Class BlockchainCache + * + * Cache blockchain information to reduce redundant RPCs to the ElectrumX servers. + */ +public class BlockchainCache { + + /** + * Keys With History + * + * Deterministic Keys with any transaction history. + */ + private Queue keysWithHistory = new ConcurrentLinkedDeque<>(); + + /** + * Transactions By Hash + * + * Transaction Hash -> Transaction + */ + private ConcurrentHashMap transactionByHash = new ConcurrentHashMap<>(); + + /** + * Key History Limit + * + * If this limit is reached, the cache will be reduced. + */ + private static final int KEY_HISTORY_LIMIT = 10000; + + /** + * Transaction Limit + * + * If this limit is reached, the cache will be cleared. + */ + private static final int TRANSACTION_LIMIT = 10000; + + /** + * Add Key With History + * + * @param key a deterministic key with a verified history + */ + public void addKeyWithHistory(DeterministicKey key) { + + if( this.keysWithHistory.size() > KEY_HISTORY_LIMIT ) this.keysWithHistory.remove(); + + this.keysWithHistory.add(key); + } + + /** + * Key Has History? + * + * @param key the deterministic key + * + * @return true if the key has a history, otherwise false + */ + public boolean keyHasHistory( DeterministicKey key ) { + return this.keysWithHistory.contains(key); + } + + /** + * Add Transaction By Hash + * + * @param hash the transaction hash + * @param transaction the transaction + */ + public void addTransactionByHash( String hash, BitcoinyTransaction transaction ) { + + if( this.transactionByHash.size() > TRANSACTION_LIMIT ) this.transactionByHash.clear(); + + this.transactionByHash.put(hash, transaction); + } + + /** + * Get Transaction By Hash + * + * @param hash the transaction hash + * + * @return the transaction, empty if the hash is not in the cache + */ + public Optional getTransactionByHash( String hash ) { + return Optional.ofNullable( this.transactionByHash.get(hash) ); + } +} \ No newline at end of file From a64e9052dd0b6ad294377c238a74f403713b2f6e Mon Sep 17 00:00:00 2001 From: kennycud Date: Wed, 4 Sep 2024 16:24:08 -0700 Subject: [PATCH 3/3] consolidated the cache limits into an attribute in Settings.java --- .../qortal/crosschain/BlockchainCache.java | 22 +++++++++---------- .../java/org/qortal/settings/Settings.java | 6 +++++ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/BlockchainCache.java b/src/main/java/org/qortal/crosschain/BlockchainCache.java index bfc6f3c6c..f6a1acf61 100644 --- a/src/main/java/org/qortal/crosschain/BlockchainCache.java +++ b/src/main/java/org/qortal/crosschain/BlockchainCache.java @@ -1,6 +1,7 @@ package org.qortal.crosschain; import org.bitcoinj.crypto.DeterministicKey; +import org.qortal.settings.Settings; import java.util.Optional; import java.util.Queue; @@ -29,18 +30,11 @@ public class BlockchainCache { private ConcurrentHashMap transactionByHash = new ConcurrentHashMap<>(); /** - * Key History Limit + * Cache Limit * - * If this limit is reached, the cache will be reduced. + * If this limit is reached, the cache will be cleared or reduced. */ - private static final int KEY_HISTORY_LIMIT = 10000; - - /** - * Transaction Limit - * - * If this limit is reached, the cache will be cleared. - */ - private static final int TRANSACTION_LIMIT = 10000; + private static final int CACHE_LIMIT = Settings.getInstance().getBlockchainCacheLimit(); /** * Add Key With History @@ -49,7 +43,9 @@ public class BlockchainCache { */ public void addKeyWithHistory(DeterministicKey key) { - if( this.keysWithHistory.size() > KEY_HISTORY_LIMIT ) this.keysWithHistory.remove(); + if( this.keysWithHistory.size() > CACHE_LIMIT ) { + this.keysWithHistory.remove(); + } this.keysWithHistory.add(key); } @@ -73,7 +69,9 @@ public boolean keyHasHistory( DeterministicKey key ) { */ public void addTransactionByHash( String hash, BitcoinyTransaction transaction ) { - if( this.transactionByHash.size() > TRANSACTION_LIMIT ) this.transactionByHash.clear(); + if( this.transactionByHash.size() > CACHE_LIMIT ) { + this.transactionByHash.clear(); + } this.transactionByHash.put(hash, transaction); } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 90dab19b2..f18ccd885 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -328,6 +328,9 @@ public class Settings { /** How many wallet keys to generate when using bitcoinj as the blockchain interface (e.g. when sending coins) */ private int bitcoinjLookaheadSize = 50; + /** How many units of data to be kept in a blockchain cache before the cache should be reduced or cleared. */ + private int blockchainCacheLimit = 1000; + // Data storage (QDN) /** Data storage enabled/disabled*/ @@ -1049,6 +1052,9 @@ public int getBitcoinjLookaheadSize() { return bitcoinjLookaheadSize; } + public int getBlockchainCacheLimit() { + return blockchainCacheLimit; + } public boolean isQdnEnabled() { return this.qdnEnabled;