🗞️ Sign transactions on different chains in parallel (#461)

.changeset/mighty-tigers-beam.md

@@ -0,0 +1,6 @@
+---
+"@layerzerolabs/ua-devtools-evm-hardhat-test": patch
+"@layerzerolabs/devtools": patch
+---
+
+Sign transactions for different chains in parallel

.changeset/three-bobcats-hear.md

@@ -0,0 +1,5 @@
+---
+"@layerzerolabs/devtools": patch
+---
+
+Add groupTransactionsByEid utility

packages/devtools/src/transactions/signer.ts

@@ -1,6 +1,7 @@
 import { createModuleLogger, pluralizeNoun, pluralizeOrdinal } from '@layerzerolabs/io-devtools'
 import type { OmniSignerFactory, OmniTransaction, OmniTransactionWithError, OmniTransactionWithReceipt } from './types'
-import { formatOmniPoint } from '@/omnigraph/format'
+import { formatEid, formatOmniPoint } from '@/omnigraph/format'
+import { groupTransactionsByEid } from './utils'
 
 export type SignAndSendResult = [
     // All the successful transactions
@@ -36,41 +37,73 @@ export const createSignAndSend =
         // Tell the user how many we are signing
         logger.debug(`Signing ${n} ${pluralizeNoun(n, 'transaction')}`)
 
-        // We'll gather the successful transactions here
-        const successful: OmniTransactionWithReceipt[] = []
-
-        for (const [index, transaction] of transactions.entries()) {
-            // We want to refer to this transaction by index, so we create an ordinal for it (1st, 2nd etc.)
-            const ordinal = pluralizeOrdinal(index + 1)
-
-            try {
-                logger.debug(`Signing ${ordinal} transaction to ${formatOmniPoint(transaction.point)}`)
-
-                logger.debug(`Creating signer for ${ordinal} transaction`)
-                const signer = await createSigner(transaction.point.eid)
-
-                logger.debug(`Signing ${ordinal} transaction`)
-                const response = await signer.signAndSend(transaction)
-
-                logger.debug(`Signed ${ordinal} transaction, got hash ${response.transactionHash}`)
-
-                const receipt = await response.wait()
-                logger.debug(`Finished ${ordinal} transaction`)
+        const transactionGroups = Array.from(groupTransactionsByEid(transactions).entries())
 
-                const result = { transaction, receipt }
-                successful.push(result)
-
-                // We'll create a clone of the successful array so that the consumers can't mutate it
-                onProgress?.(result, [...successful])
-            } catch (error) {
-                logger.debug(`Failed to process ${ordinal} transaction: ${error}`)
-
-                return [successful, [{ transaction, error }], transactions.slice(index)]
-            }
-        }
-
-        // Tell the inquisitive user what a good job we did
-        logger.debug(`Successfully signed ${n} ${pluralizeNoun(n, 'transaction')}`)
-
-        return [successful, [], []]
+        // We'll gather the state of the signing here
+        const successful: OmniTransactionWithReceipt[] = []
+        const errors: OmniTransactionWithError[] = []
+
+        await Promise.allSettled(
+            transactionGroups.map(async ([eid, eidTransactions]): Promise<void> => {
+                const eidName = formatEid(eid)
+
+                logger.debug(
+                    `Signing ${eidTransactions.length} ${pluralizeNoun(eidTransactions.length, 'transaction')} for ${eidName}`
+                )
+
+                logger.debug(`Creating signer for ${eidName}`)
+                const signer = await createSigner(eid)
+
+                for (const [index, transaction] of eidTransactions.entries()) {
+                    // We want to refer to this transaction by index so we create an ordinal for it (1st, 2nd etc)
+                    const ordinal = pluralizeOrdinal(index + 1)
+
+                    try {
+                        logger.debug(
+                            `Signing ${ordinal} transaction for ${eidName} to ${formatOmniPoint(transaction.point)}`
+                        )
+                        const response = await signer.signAndSend(transaction)
+
+                        logger.debug(
+                            `Signed ${ordinal} transaction for ${eidName}, got hash ${response.transactionHash}`
+                        )
+
+                        const receipt = await response.wait()
+                        logger.debug(`Finished ${ordinal} transaction for ${eidName}`)
+
+                        const result: OmniTransactionWithReceipt = { transaction, receipt }
+
+                        // Here we want to update the global state of the signing
+                        successful.push(result)
+
+                        // We'll create a clone of the successful array so that the consumers can't mutate it
+                        onProgress?.(result, [...successful])
+                    } catch (error) {
+                        logger.debug(`Failed to process ${ordinal} transaction for ${eidName}: ${error}`)
+
+                        // Update the error state
+                        errors.push({ transaction, error })
+
+                        // We want to stop the moment we hit an error
+                        return
+                    }
+                }
+
+                // Tell the inquisitive user what a good job we did
+                logger.debug(`Successfully signed ${n} ${pluralizeNoun(n, 'transaction')} for ${eidName}`)
+            })
+        )
+
+        // Now we create a list of the transactions that have not been touched
+        //
+        // We do this by taking all the transactions, then filtering out those
+        // that don't have a result associated with them
+        //
+        // This functionality relies on reference equality of the transactions objects
+        // so it's important that we don't mess with those and push the transaction
+        // objects directly to the `successful` and `errors` arrays, without any rest spreading or whatnot
+        const processed = new Set<OmniTransaction>(successful.map(({ transaction }) => transaction))
+        const pending = transactions.filter((transaction) => !processed.has(transaction))
+
+        return [successful, errors, pending]
     }

packages/devtools/src/transactions/utils.ts

@@ -1,7 +1,24 @@
-import { OmniTransaction } from './types'
+import type { EndpointId } from '@layerzerolabs/lz-definitions'
+import type { OmniTransaction } from './types'
 
 const isNonNullable = <T>(value: T | null | undefined): value is T => value != null
 
 export const flattenTransactions = (
     transations: (OmniTransaction | OmniTransaction[] | null | undefined)[]
 ): OmniTransaction[] => transations.filter(isNonNullable).flat()
+
+/**
+ * Groups transactions by their `eid`, preserving the order per group
+ *
+ * @param {OmniTransaction[]} transactions
+ * @returns {Map<EndpointId, OmniTransaction[]>}
+ */
+export const groupTransactionsByEid = (transactions: OmniTransaction[]): Map<EndpointId, OmniTransaction[]> =>
+    transactions.reduce(
+        (transactionsByEid, transaction) =>
+            transactionsByEid.set(transaction.point.eid, [
+                ...(transactionsByEid.get(transaction.point.eid) ?? []),
+                transaction,
+            ]),
+        new Map<EndpointId, OmniTransaction[]>()
+    )

packages/devtools/test/transactions/signer.test.ts

@@ -1,6 +1,12 @@
 import fc from 'fast-check'
 import { pointArbitrary } from '@layerzerolabs/test-devtools'
-import { OmniSignerFactory, OmniTransaction, OmniTransactionResponse, createSignAndSend } from '@/transactions'
+import {
+    OmniSignerFactory,
+    OmniTransaction,
+    OmniTransactionResponse,
+    createSignAndSend,
+    groupTransactionsByEid,
+} from '@/transactions'
 
 describe('transactions/signer', () => {
     const transactionArbitrary: fc.Arbitrary<OmniTransaction> = fc.record({
@@ -44,10 +50,18 @@ describe('transactions/signer', () => {
                     // Now we send all the transactions to the flow and observe the output
                     const [successful, errors, pending] = await signAndSendTransactions(transactions)
 
-                    expect(successful).toEqual(transactions.map((transaction) => ({ transaction, receipt })))
+                    // Since we are executing groups of transactions in parallel,
+                    // in general the order of successful transaction will not match the order of input transactions
+                    expect(successful).toContainAllValues(transactions.map((transaction) => ({ transaction, receipt })))
                     expect(errors).toEqual([])
                     expect(pending).toEqual([])
 
+                    // What needs to match though is the order of successful transactions within groups
+                    //
+                    // For that we group the successful transactions and make sure those are equal to the grouped original transactions
+                    const groupedSuccessful = groupTransactionsByEid(successful.map(({ transaction }) => transaction))
+                    expect(groupedSuccessful).toEqual(groupTransactionsByEid(transactions))
+
                     // We also check that the signer factory has been called with the eids
                     for (const transaction of transactions) {
                         expect(signerFactory).toHaveBeenCalledWith(transaction.point.eid)
@@ -93,6 +107,17 @@ describe('transactions/signer', () => {
                             [failedTransaction, Promise.resolve(unsuccessfulResponse)],
                         ])
 
+                        const expectedSuccessful = [
+                            // The first batch should all go through
+                            ...firstBatch,
+                            // The transactions that are not on the chain affected by the failed transaction should also pass
+                            ...secondBatch.filter(({ point }) => point.eid !== failedTransaction.point.eid),
+                        ]
+
+                        const expectedPending = secondBatch.filter(
+                            ({ point }) => point.eid === failedTransaction.point.eid
+                        )
+
                         // Our signAndSend will then use the map to resolve/reject transactions
                         const signAndSend = jest.fn().mockImplementation((t) => implementations.get(t))
                         const sign = jest.fn().mockRejectedValue('Oh god no')
@@ -103,9 +128,21 @@ describe('transactions/signer', () => {
                         const transactions = [...firstBatch, failedTransaction, ...secondBatch]
                         const [successful, errors, pending] = await signAndSendTransactions(transactions)
 
-                        expect(successful).toEqual(firstBatch.map((transaction) => ({ transaction, receipt })))
+                        // Since we are executing groups of transactions in parallel,
+                        // in general the order of successful transaction will not match the order of input transactions
+                        expect(successful).toContainAllValues(
+                            expectedSuccessful.map((transaction) => ({ transaction, receipt }))
+                        )
                         expect(errors).toEqual([{ transaction: failedTransaction, error }])
-                        expect(pending).toEqual([failedTransaction, ...secondBatch])
+                        expect(pending).toEqual([failedTransaction, ...expectedPending])
+
+                        // What needs to match though is the order of successful transactions within groups
+                        //
+                        // For that we group the successful transactions and make sure those are equal to the grouped original transactions
+                        const groupedSuccessful = groupTransactionsByEid(
+                            successful.map(({ transaction }) => transaction)
+                        )
+                        expect(groupedSuccessful).toEqual(groupTransactionsByEid(expectedSuccessful))
 
                         // We also check that the signer factory has been called with the eids
                         expect(signerFactory).toHaveBeenCalledWith(failedTransaction.point.eid)
@@ -145,6 +182,17 @@ describe('transactions/signer', () => {
                             [failedTransaction, Promise.reject(error)],
                         ])
 
+                        const expectedSuccessful = [
+                            // The first batch should all go through
+                            ...firstBatch,
+                            // The transactions that are not on the chain affected by the failed transaction should also pass
+                            ...secondBatch.filter(({ point }) => point.eid !== failedTransaction.point.eid),
+                        ]
+
+                        const expectedPending = secondBatch.filter(
+                            ({ point }) => point.eid === failedTransaction.point.eid
+                        )
+
                         // Our signAndSend will then use the map to resolve/reject transactions
                         const signAndSend = jest.fn().mockImplementation((t) => implementations.get(t))
                         const sign = jest.fn().mockRejectedValue('Oh god no')
@@ -155,9 +203,21 @@ describe('transactions/signer', () => {
                         const transactions = [...firstBatch, failedTransaction, ...secondBatch]
                         const [successful, errors, pending] = await signAndSendTransactions(transactions)
 
-                        expect(successful).toEqual(firstBatch.map((transaction) => ({ transaction, receipt })))
+                        // Since we are executing groups of transactions in parallel,
+                        // in general the order of successful transaction will not match the order of input transactions
+                        expect(successful).toContainAllValues(
+                            expectedSuccessful.map((transaction) => ({ transaction, receipt }))
+                        )
                         expect(errors).toEqual([{ transaction: failedTransaction, error }])
-                        expect(pending).toEqual([failedTransaction, ...secondBatch])
+                        expect(pending).toEqual([failedTransaction, ...expectedPending])
+
+                        // What needs to match though is the order of successful transactions within groups
+                        //
+                        // For that we group the successful transactions and make sure those are equal to the grouped original transactions
+                        const groupedSuccessful = groupTransactionsByEid(
+                            successful.map(({ transaction }) => transaction)
+                        )
+                        expect(groupedSuccessful).toEqual(groupTransactionsByEid(expectedSuccessful))
 
                         // We also check that the signer factory has been called with the eids
                         expect(signerFactory).toHaveBeenCalledWith(failedTransaction.point.eid)
@@ -190,16 +250,16 @@ describe('transactions/signer', () => {
                     const signAndSendTransactions = createSignAndSend(signerFactory)
 
                     const handleProgress = jest.fn()
-                    await signAndSendTransactions(transactions, handleProgress)
+                    const [successful] = await signAndSendTransactions(transactions, handleProgress)
 
                     // We check whether onProgress has been called for every transaction
-                    for (const [index, transaction] of transactions.entries()) {
+                    for (const [index, transaction] of successful.entries()) {
                         expect(handleProgress).toHaveBeenNthCalledWith(
                             index + 1,
                             // We expect the transaction in question to be passed
-                            { transaction, receipt },
+                            transaction,
                             // As well as the list of all the successful transactions so far
-                            transactions.slice(0, index + 1).map((transaction) => ({ transaction, receipt }))
+                            successful.slice(0, index + 1)
                         )
                     }
                 })

packages/devtools/test/transactions/utils.test.ts

@@ -1,6 +1,6 @@
 import fc from 'fast-check'
 import { pointArbitrary } from '@layerzerolabs/test-devtools'
-import { OmniTransaction, flattenTransactions } from '@/transactions'
+import { OmniTransaction, flattenTransactions, groupTransactionsByEid } from '@/transactions'
 
 describe('transactions/utils', () => {
     const nullableArbitrary = fc.constantFrom(null, undefined)
@@ -50,4 +50,46 @@ describe('transactions/utils', () => {
             )
         })
     })
+
+    describe('groupTransactionsByEid', () => {
+        it('should return an empty map when an empty array is passed in', () => {
+            expect(groupTransactionsByEid([])).toEqual(new Map())
+        })
+
+        it('should return a map with containing all the transactions passed in', () => {
+            fc.assert(
+                fc.property(fc.array(transactionArbitrary), (transactions) => {
+                    const grouped = groupTransactionsByEid(transactions)
+
+                    for (const transaction of transactions) {
+                        expect(grouped.get(transaction.point.eid)).toContain(transaction)
+                    }
+                })
+            )
+        })
+
+        it('should preserve the order of transaction per group', () => {
+            fc.assert(
+                fc.property(fc.array(transactionArbitrary), (transactions) => {
+                    const grouped = groupTransactionsByEid(transactions)
+
+                    for (const transactionsForEid of grouped.values()) {
+                        // Here we want to make sure that within a group of transactions,
+                        // no transactions have changed order
+                        //
+                        // The logic here goes something like this:
+                        // - We look for the indices of transactions from the grouped array in the original array
+                        // - If two transactions swapped places, this array would then contain an inversion
+                        //   (transaction at an earlier index in the grouped array would appear after a transaction
+                        //   at a later index). So what we do is remove the inversions by sorting the array of indices
+                        //   and make sure this sorted array matches the original one
+                        const transactionIndices = transactionsForEid.map((t) => transactions.indexOf(t))
+                        const sortedTransactionIndices = transactionIndices.slice().sort()
+
+                        expect(transactionIndices).toEqual(sortedTransactionIndices)
+                    }
+                })
+            )
+        })
+    })
 })

tests/ua-devtools-evm-hardhat-test/test/oapp/config.test.ts

@@ -585,7 +585,7 @@ describe('oapp/config', () => {
         })
 
         describe('configureConfig configureSendConfig and configureReceiveConfig separately', () => {
-            let bscContract: OmniContract, bscPoint: OmniPoint, bscOAppSdk: OApp
+            let bscContract: OmniContract, bscPoint: OmniPoint, bscOAppSdk: IOApp
 
             beforeEach(async () => {
                 bscContract = await contractFactory(bscPointHardhat)