From e17ad214acea16c498a5bccaa554b3aab16ac488 Mon Sep 17 00:00:00 2001 From: Duddino Date: Thu, 17 Oct 2024 21:34:33 +0200 Subject: [PATCH 01/24] Add faster shield sync --- scripts/wallet.js | 118 +++++++++++++++++++++++++++++++--------------- 1 file changed, 79 insertions(+), 39 deletions(-) diff --git a/scripts/wallet.js b/scripts/wallet.js index 853c15ae9..7d99178a9 100644 --- a/scripts/wallet.js +++ b/scripts/wallet.js @@ -1,6 +1,6 @@ import { validateMnemonic } from 'bip39'; import { decrypt } from './aes-gcm.js'; -import { parseWIF } from './encoding.js'; +import { bytesToNum, mergeUint8Arrays, parseWIF } from './encoding.js'; import { beforeUnloadListener, blockCount } from './global.js'; import { getNetwork } from './network.js'; import { MAX_ACCOUNT_GAP, SHIELD_BATCH_SYNC_SIZE } from './chain_params.js'; @@ -721,48 +721,88 @@ export class Wallet { if (!this.#shield || this.#isSynced) { return; } - const cNet = getNetwork(); + try { - const blockHeights = (await cNet.getShieldBlockList()).filter( - (b) => b > this.#shield.getLastSyncedBlock() + const req = await fetch( + `http://127.0.0.1:3000/mainnet/getshielddata?startBlock=${ + wallet.#shield.getLastSyncedBlock() + 1 + }` + ); + const reader = req.body.getReader(); + /** @type {number[]} Array of bytes that we are processing **/ + let processing = []; + /** @type{string[]} Array of txs in the current block */ + let txs = []; + let processedBytes = 0; + const length = req.headers.get('Content-Length'); + getEventEmitter().emit( + 'shield-sync-status-update', + 0, + length, + false ); - const batchSize = SHIELD_BATCH_SYNC_SIZE; - let handled = 0; - let downloaded = 0; - const blocks = []; - let syncing = false; - await startBatch( - async (i) => { - let block; - block = await cNet.getBlock(blockHeights[i], true); - downloaded++; - blocks[i] = block; - // We need to process blocks monotically - // When we get a block, start from the first unhandled - // One and handle as many as possible - for (let j = handled; blocks[j]; j = handled) { - if (syncing) break; - syncing = true; - handled++; - await this.#shield.handleBlock(blocks[j]); - // Backup every 500 handled blocks - if (handled % 500 == 0) await this.saveShieldOnDisk(); - // Delete so we don't have to hold all blocks in memory - // until we finish syncing - delete blocks[j]; - syncing = false; + let i = 0; + while (true) { + /** + * @type {{done: boolean, value: Uint8Array?}} + */ + const { done, value } = await reader.read(); + /** + * Array of blocks ready to pass to the shield library + * @type {{txs: string[]; height: number; time: number}[]} + */ + const blocksArray = []; + + if (value) { + // Append received bytes in the processing array + value.forEach((v) => processing.push(v)); + processedBytes += value.length; + // Loop until we have less than 4 bytes (length) + while (processing.length - i >= 4) { + const length = Number( + bytesToNum(processing.slice(i, i + 4)) + ); + // If we have less bytes than the length break and wait for the next + // batch of bytes + if (processing.length - i < length) break; + + i += 4; + const bytes = processing.slice(i, length + i); + i += length; + // 0x5d rapresents the block + if (bytes[0] === 0x5d) { + const height = Number( + bytesToNum(bytes.slice(1, 5)) + ); + const time = Number(bytesToNum(bytes.slice(9, 13))); + + blocksArray.push({ txs, height, time }); + txs = []; + } else if (bytes[0] == 0x03) { + // 0x03 is the tx version. We should only get v3 transactions + const hex = bytesToHex(bytes); + txs.push(hex); + } else { + // This is neither a block or a tx. + throw new Error('Failed to parse shield binary'); + } } + console.log(processing.length - i); + } + + console.log(blocksArray); + // Process the current batch of blocks before starting to parse the next one + await this.#shield.handleBlocks(blocksArray); + // Emit status update + getEventEmitter().emit( + 'shield-sync-status-update', + processedBytes, + length, + false + ); + if (done) break; + } - getEventEmitter().emit( - 'shield-sync-status-update', - downloaded - 1, - blockHeights.length, - false - ); - }, - blockHeights.length, - batchSize - ); getEventEmitter().emit('shield-sync-status-update', 0, 0, true); } catch (e) { debugError(DebugTopics.WALLET, e); From 3de3aec610536a2862faf7b175d0bff88e3d384b Mon Sep 17 00:00:00 2001 From: Duddino Date: Thu, 17 Oct 2024 21:48:39 +0200 Subject: [PATCH 02/24] Change shield progress text to use bytes --- scripts/dashboard/WalletBalance.vue | 33 +++++++++++------------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/scripts/dashboard/WalletBalance.vue b/scripts/dashboard/WalletBalance.vue index c254400ef..c6c14b9a2 100644 --- a/scripts/dashboard/WalletBalance.vue +++ b/scripts/dashboard/WalletBalance.vue @@ -63,13 +63,12 @@ const { // Transparent sync status const transparentSyncing = ref(false); -const transparentProgressSyncing = ref(0.0); +const percentage = ref(0.0); const syncTStr = ref(''); // Shield sync status const shieldSyncing = ref(false); -const shieldPercentageSyncing = ref(0.0); -const shieldBlockRemainingSyncing = ref(0); +const shieldSyncingStr = ref(''); // Shield transaction creation const isCreatingTx = ref(false); @@ -135,20 +134,20 @@ getEventEmitter().on( 'transparent-sync-status-update', (str, progress, finished) => { syncTStr.value = str; - transparentProgressSyncing.value = progress; + percentage.value = progress; transparentSyncing.value = !finished; } ); getEventEmitter().on( 'shield-sync-status-update', - (blocks, totalBlocks, finished) => { - shieldPercentageSyncing.value = Math.round( - (blocks / totalBlocks) * 100 - ); - shieldBlockRemainingSyncing.value = ( - totalBlocks - blocks - ).toLocaleString('en-GB'); + (bytes, totalBytes, finished) => { + percentage.value = Math.round((100 * bytes) / totalBytes); + const mb = bytes / 1_000_000; + const totalMb = totalBytes / 1_000_000; + shieldSyncingStr.value = `Syncing Shield (${mb.toFixed( + 2 + )}MB/${totalMb.toFixed(2)}MB)...`; shieldSyncing.value = !finished; } ); @@ -507,18 +506,10 @@ function restoreWallet() {
- {{ - transparentSyncing - ? syncTStr - : `Syncing ${shieldBlockRemainingSyncing} Blocks...` - }} + {{ transparentSyncing ? syncTStr : shieldSyncingStr }} Date: Mon, 21 Oct 2024 10:06:47 +0200 Subject: [PATCH 05/24] Remove console.logs --- scripts/wallet.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/wallet.js b/scripts/wallet.js index 7d99178a9..51d5af01b 100644 --- a/scripts/wallet.js +++ b/scripts/wallet.js @@ -787,10 +787,8 @@ export class Wallet { throw new Error('Failed to parse shield binary'); } } - console.log(processing.length - i); } - console.log(blocksArray); // Process the current batch of blocks before starting to parse the next one await this.#shield.handleBlocks(blocksArray); // Emit status update From e5aba37c83ab99ddf283981e490b3a6485cf0249 Mon Sep 17 00:00:00 2001 From: Duddino Date: Mon, 21 Oct 2024 10:11:42 +0200 Subject: [PATCH 06/24] Fix merge error --- scripts/dashboard/WalletBalance.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dashboard/WalletBalance.vue b/scripts/dashboard/WalletBalance.vue index 1b4c14742..d87b18d7d 100644 --- a/scripts/dashboard/WalletBalance.vue +++ b/scripts/dashboard/WalletBalance.vue @@ -1,6 +1,6 @@