Add faster shield sync

scripts/dashboard/WalletBalance.vue

@@ -60,13 +60,12 @@ const {
 
 // Transparent sync status
 const transparentSyncing = ref(false);
-const transparentProgressSyncing = ref(0.0);
+const percentage = ref(0.0);
 const syncTStr = ref('');
 
 // Shield sync status
 const shieldSyncing = ref(false);
-const shieldPercentageSyncing = ref(0.0);
-const shieldBlockRemainingSyncing = ref(0);
+const shieldSyncingStr = ref('');
 
 // Shield transaction creation
 const isCreatingTx = ref(false);
@@ -137,20 +136,20 @@ getEventEmitter().on(
         ]);
         const progress = ((totalPages - i) / totalPages) * 100;
         syncTStr.value = str;
-        transparentProgressSyncing.value = progress;
+        percentage.value = progress;
         transparentSyncing.value = !finished;
     }
 );
 
 getEventEmitter().on(
     'shield-sync-status-update',
-    (blocks, totalBlocks, finished) => {
-        shieldPercentageSyncing.value = Math.round(
-            (blocks / totalBlocks) * 100
-        );
-        shieldBlockRemainingSyncing.value = (
-            totalBlocks - blocks
-        ).toLocaleString('en-GB');
+    (bytes, totalBytes, finished) => {
+        percentage.value = Math.round((100 * bytes) / totalBytes);
+        const mb = bytes / 1_000_000;
+        const totalMb = totalBytes / 1_000_000;
+        shieldSyncingStr.value = `Syncing Shield (${mb.toFixed(
+            1
+        )}MB/${totalMb.toFixed(1)}MB)`;
         shieldSyncing.value = !finished;
     }
 );
@@ -510,18 +509,10 @@ function restoreWallet() {
                     <i class="fas fa-spinner spinningLoading"></i>
                 </div>
                 <div style="width: 100%">
-                    {{
-                        transparentSyncing
-                            ? syncTStr
-                            : `Syncing ${shieldBlockRemainingSyncing} Blocks...`
-                    }}
+                    {{ transparentSyncing ? syncTStr : shieldSyncingStr }}
                     <LoadingBar
                         :show="true"
-                        :percentage="
-                            transparentSyncing
-                                ? transparentProgressSyncing
-                                : shieldPercentageSyncing
-                        "
+                        :percentage="percentage"
                         style="
                             border: 1px solid #932ecd;
                             border-radius: 4px;

scripts/network/network.js

@@ -101,6 +101,10 @@
         throw new Error('getProposals must be implemented');
     }
 
+    async getShieldData(_initialBlock = 0) {
+        throw new Error('getShieldData must be implemented');
+    }
+
     async getProposalVote(_proposalName, _collateralTxId, _outidx) {
         throw new Error('getProposalVote must be implemented');
     }
@@ -117,9 +121,9 @@
     }
 
     async submitProposal({
         name,
         url,
         nPayments,
         start,
         address,
         monthlyPayment,
@@ -200,7 +204,7 @@
         // Use Nodes as a fallback
         let strTXID = await this.#callRPC(
             '/sendrawtransaction?params=' + hex,
-            true
+            'text'
         );
         strTXID = strTXID.replace(/"/g, '');
         return { result: strTXID };
@@ -316,6 +320,14 @@
             true
         );
     }
+
+    async getShieldData(startBlock) {
+        const res = await this.#fetchNode(
+            `/getshielddata?startBlock=${startBlock}`
+        );
+        if (!res.ok) throw new Error('Invalid response');
+        return res;
+    }
 }
 
 /**

scripts/network/network_manager.js

@@ -259,6 +259,10 @@ class NetworkManager {
         );
     }
 
+    async getShieldData(initialBlock = 0) {
+        return await this.#retryWrapper('getShieldData', true, initialBlock);
+    }
+
     /**
      * Submit a proposal
      * @param {Object} options

scripts/transaction.js

@@ -145,15 +145,26 @@ export class Transaction {
         return this.version >= SAPLING_TX_VERSION;
     }
 
+    /**
+     * @returns {string} txid of the transaction
+     */
     get txid() {
         if (!this.__original.#txid) {
-            this.__original.#txid = bytesToHex(
-                dSHA256(hexToBytes(this.serialize())).reverse()
+            this.__original.#txid = Transaction.getTxidFromHex(
+                this.serialize()
             );
         }
         return this.__original.#txid;
     }
 
+    /**
+     * @param {string} hex - Hex encoded transaction
+     * @returns {string} txid
+     */
+    static getTxidFromHex(hex) {
+        return bytesToHex(dSHA256(hexToBytes(hex)).reverse());
+    }
+
     get hasShieldData() {
         return this.bindingSig !== '';
     }

scripts/wallet.js

@@ -1,6 +1,6 @@
 import { validateMnemonic } from 'bip39';
 import { decrypt } from './aes-gcm.js';
-import { parseWIF } from './encoding.js';
+import { bytesToNum, mergeUint8Arrays, parseWIF } from './encoding.js';
 import { beforeUnloadListener, blockCount } from './global.js';
 import { getNetwork } from './network/network_manager.js';
 import { MAX_ACCOUNT_GAP, SHIELD_BATCH_SYNC_SIZE } from './chain_params.js';
@@ -34,6 +34,7 @@ import { guiToggleReceiveType } from './contacts-book.js';
 import { TransactionBuilder } from './transaction_builder.js';
 import { createAlert } from './alerts/alert.js';
 import { AsyncInterval } from './async_interval.js';
+import { cNode } from './settings.js';
 import { debugError, debugLog, DebugTopics } from './debug.js';
 import { OrderedArray } from './ordered_array.js';
 
@@ -772,48 +773,95 @@ export class Wallet {
         if (!this.#shield || this.#isSynced) {
             return;
         }
-        const cNet = getNetwork();
+
         try {
-            const blockHeights = (await cNet.getShieldBlockList()).filter(
-                (b) => b > this.#shield.getLastSyncedBlock()
+            const network = getNetwork();
+            const req = await network.getShieldData(
+                wallet.#shield.getLastSyncedBlock() + 1
             );
-            const batchSize = SHIELD_BATCH_SYNC_SIZE;
-            let handled = 0;
-            let downloaded = 0;
-            const blocks = [];
-            let syncing = false;
-            await startBatch(
-                async (i) => {
-                    let block;
-                    block = await cNet.getBlock(blockHeights[i]);
-                    downloaded++;
-                    blocks[i] = block;
-                    // We need to process blocks monotically
-                    // When we get a block, start from the first unhandled
-                    // One and handle as many as possible
-                    for (let j = handled; blocks[j]; j = handled) {
-                        if (syncing) break;
-                        syncing = true;
-                        handled++;
-                        await this.#shield.handleBlock(blocks[j]);
-                        // Backup every 500 handled blocks
-                        if (handled % 500 === 0) await this.saveShieldOnDisk();
-                        // Delete so we don't have to hold all blocks in memory
-                        // until we finish syncing
-                        delete blocks[j];
-                        syncing = false;
+            if (!req.ok) throw new Error("Couldn't sync shield");
+            const reader = req.body.getReader();
+
+            /** @type{string[]} Array of txs in the current block */
+            let txs = [];
+            let processedBytes = 0;
+            const length = req.headers.get('Content-Length');
+            /** @type {Uint8Array} Array of bytes that we are processing **/
+            const processing = new Uint8Array(length);
+            getEventEmitter().emit(
+                'shield-sync-status-update',
+                0,
+                length,
+                false
+            );
+            let i = 0;
+            let max = 0;
+            while (true) {
+                /**
+                 * @type {{done: boolean, value: Uint8Array?}}
+                 */
+                const { done, value } = await reader.read();
+                /**
+                 * Array of blocks ready to pass to the shield library
+                 * @type {{txs: string[]; height: number; time: number}[]}
+                 */
+                const blocksArray = [];
+
+                if (value) {
+                    // Append received bytes in the processing array
+                    processing.set(value, max);
+                    max += value.length;
+                    // value.forEach((v) => processing.push(v));
+                    processedBytes += value.length;
+                    // Loop until we have less than 4 bytes (length)
+                    while (max - i >= 4) {
+                        const length = Number(
+                            bytesToNum(processing.subarray(i, i + 4))
+                        );
+                        // If we have less bytes than the length break and wait for the next
+                        // batch of bytes
+                        if (max - i < length) break;
+
+                        i += 4;
+                        const bytes = processing.subarray(i, length + i);
+                        i += length;
+                        // 0x5d rapresents the block
+                        if (bytes[0] === 0x5d) {
+                            const height = Number(
+                                bytesToNum(bytes.slice(1, 5))
+                            );
+                            const time = Number(bytesToNum(bytes.slice(9, 13)));
+
+                            blocksArray.push({ txs, height, time });
+                            txs = [];
+                        } else if (bytes[0] === 0x03) {
+                            // 0x03 is the tx version. We should only get v3 transactions
+                            const hex = bytesToHex(bytes);
+                            txs.push({
+                                hex,
+                                txid: Transaction.getTxidFromHex(hex),
+                            });
+                        } else {
+                            // This is neither a block or a tx.
+                            throw new Error('Failed to parse shield binary');
+                        }
                     }
+                }
+
+                // Process the current batch of blocks before starting to parse the next one
+                if (blocksArray.length) {
+                    await this.#shield.handleBlocks(blocksArray);
+                }
+                // Emit status update
+                getEventEmitter().emit(
+                    'shield-sync-status-update',
+                    processedBytes,
+                    length,
+                    false
+                );
+                if (done) break;
+            }
 
-                    getEventEmitter().emit(
-                        'shield-sync-status-update',
-                        downloaded - 1,
-                        blockHeights.length,
-                        false
-                    );
-                },
-                blockHeights.length,
-                batchSize
-            );
             getEventEmitter().emit('shield-sync-status-update', 0, 0, true);
         } catch (e) {
             debugError(DebugTopics.WALLET, e);