Merge branch 'main' into teohaik/add-releaseit-tool

.github/workflows/node.js.yml

@@ -15,14 +15,15 @@ jobs:
     steps:
       - name: Dump Environment Variables
         env:
+          ADMIN_ADDRESS: ${{vars.ADMIN_ADDRESS}}
           ADMIN_SECRET_KEY: ${{ secrets.ADMIN_SECRET_KEY }}
           TEST_USER_SECRET: ${{ secrets.TEST_USER_SECRET }}
           TEST_USER_ADDRESS: ${{ vars.TEST_USER_ADDRESS }}
           NFT_APP_PACKAGE_ID: ${{ vars.NFT_APP_PACKAGE_ID }}
           NFT_APP_ADMIN_CAP: ${{ vars.NFT_APP_ADMIN_CAP }}
           SUI_NODE: ${{ vars.SUI_NODE }}
           GET_WORKER_TIMEOUT_MS: ${{ vars.GET_WORKER_TIMEOUT_MS }}
-        run: echo "SUI_NODE=$SUI_NODE | GET_WORKER_TIMEOUT_MS=$GET_WORKER_TIMEOUT_MS | NFT_APP_PACKAGE_ID=$NFT_APP_PACKAGE_ID | NFT_APP_ADMIN_CAP=$NFT_APP_ADMIN_CAP | TEST_USER_ADDRESS=$TEST_USER_ADDRESS | TEST_NFT_OBJECT_ID=$TEST_NFT_OBJECT_ID | TEST_NON_EXISTING_OBJECT_ID=$TEST_NON_EXISTING_OBJECT_ID | TEST_NOT_OWNED_BY_ADMIN_OBJECT_ID=$TEST_NOT_OWNED_BY_ADMIN_OBJECT_ID "
+        run: echo "SUI_NODE=$SUI_NODE | GET_WORKER_TIMEOUT_MS=$GET_WORKER_TIMEOUT_MS | NFT_APP_PACKAGE_ID=$NFT_APP_PACKAGE_ID | NFT_APP_ADMIN_CAP=$NFT_APP_ADMIN_CAP | TEST_USER_ADDRESS=$TEST_USER_ADDRESS |  ADMIN_ADDRESS=$ADMIN_ADDRESS "
       - name: Dump GitHub context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
@@ -48,6 +49,7 @@ jobs:
     - run: npm run lint
     - run: npm run test
       env:
+        ADMIN_ADDRESS: ${{vars.ADMIN_ADDRESS}}
         ADMIN_SECRET_KEY: ${{ secrets.ADMIN_SECRET_KEY }}
         TEST_USER_SECRET: ${{ secrets.TEST_USER_SECRET }}
         TEST_USER_ADDRESS: ${{ vars.TEST_USER_ADDRESS }}

README.md

@@ -1,14 +1,139 @@
 # Sui Execution Handler
 
-This is an initial design of a coin management system. The outcome should be a library with a good docs that will allow any builder to have an address issuing many concurrent transactions without locking its coins.
+A library that provides a set of tools for managing multiple concurrent
+transactions on the Sui network to help avoiding object equivocation and locking.
+
+## Quickstart
+Note: You can find a more detailed example in a later section bellow.
+```typescript
+// Initialize the ExecutorServiceHandler.
+const eshandler = await ExecutorServiceHandler.initialize(
+adminKeypair,
+suiClient,
+);
+
+/// An then for each incoming request...
+/// ...get the incoming transactionBlock
+const myTransactionBlock;
+
+/// and send it for execution
+const promise = eshandler.execute(myTransactionBlock, suiClient, splitStrategy);
+```
+
+## Motivation
+
+**Equivocation** is a common pitfall for builders using owned objects:
+Implementing horizontal scaling or concurrency for a service that executes
+transactions on Sui in the natural way results in an architecture that issues 
+multiple transactions in parallel from the same account.
+
+The community largely avoids using owned objects as a result, which also means 
+they don’t benefit from their lower latency, which is a **unique selling point** 
+for Sui. On top of that, they are impossible to completely avoid, because the 
+transaction’s gas coin must be owned.
+
+Finally, the situation is exacerbated by **gas smashing** (which combines automatically
+all transaction’s gas coins into one) and our SDK’s default **coin selection** logic
+which uses all the `Coin<SUI>`s owned by an address for every transaction’s 
+gas payment. These defaults make sending transactions from an individual’s wallet 
+simple (doing so automatically cleans up Coin dust), but mean that developers 
+writing services need to work against the defaults to maintain distinct gas 
+coins to run transactions in parallel.
+
+**This library is a solution to the above, simplifying access to owned objects from 
+back-end services that also need to take advantage of concurrency,
+without equivocating their objects.**
+
+## Solution
+
+The main modules of the library are `executorServiceHandler.ts` and `pool.ts`.
+
+- `executorServiceHandler.ts` contains the logic of the executor service - meaning
+that it acts like a load balancer, distributing the transactions to the worker pools.
+- `pool.ts` contains the logic of the worker pools.
+
+
+As a user of the library you will only need to use the `executorServiceHandler.ts` module.
+
+The basic idea of our solution is to use multiple **worker pools**
+where each one of them will execute one of the transactions. 
+
+The flow goes as follows:
+
+1. First we initialize the ExecutorServiceHandler containing only one mainPool.
+Then whenever a transaction is submitted to the ExecutorServiceHandler, it will
+try to find if there is an available worker pool to sign and execute the transaction. 
+Note that the main pool is not a worker pool.
+
+2. If a worker pool is not found, the executor handler will create one by splitting
+the mainPool - i.e. taking a part of the mainPool's objects and coins and creating a new worker pool.  
+This is how the executor handler scales up. You can define the split logic by providing
+a SplitStrategy object to the ExecutorServiceHandler on initialization.
+
+### Example code
+
+Let's define an example to make things clearer: Assume that we need to execute 10 transactions that transfer 100 MIST each to a fixed recipient.
+```typescript
+/* HERE ARE DEFINED THE PREPARATORY STEPS IF YOU WANT TO CODE ALONG*/
+// Define the transaction block
+function createPaymentTxb(recipient: string): TransactionBlock {
+  const txb = new TransactionBlock();
+  const [coin] = txb.splitCoins(txb.gas, [txb.pure(MIST_TO_TRANSFER)]);
+  txb.transferObjects([coin], txb.pure("<recipient-address>"));
+  return txb;
+}
+// Define your admin keypair and client
+const ADMIN_SECRET_KEY: string = "<your-address-secret-key>";
+const adminPrivateKeyArray = Uint8Array.from(
+  Array.from(fromB64(ADMIN_SECRET_KEY)),
+);
+const adminKeypair = Ed25519Keypair.fromSecretKey(
+  adminPrivateKeyArray.slice(1),
+);
+
+const client = new SuiClient({
+  url: process.env.SUI_NODE!,
+});
+
+```
+
+Now we setup the service handler and to execute the transactions we defined above, we will use the `execute` method of the `ExecutorServiceHandler` class.
+
+```typescript
+// Setup the executor service
+const eshandler = await ExecutorServiceHandler.initialize(
+  adminKeypair,
+  client,
+);
+// Define the number of transactions to execute
+const promises = [];
+let txb: TransactionBlock;
+for (let i = 0; i < 10; i++) {
+  txb = createPaymentTxb(process.env.TEST_USER_ADDRESS!);
+  promises.push(eshandler.execute(txb, client, splitStrategy));
+}
+
+// Collect the promise results
+const results = await Promise.allSettled(promises);
+```
+
+It's that simple! 
+
+## Development
 
-## Develop
+### Installing the library
 
 Install dependencies with `npm install`
 
-### Test
+### Code consistency
+Before commiting your changes, run `npm run lint` to check for code style consistency.
+
+### Testing
+
+Tests are a great way to get familiar with the library. For each test scenario
+there is a small description of the test's purpose and the library's commands to achieve that.
 
-To setup the tests environment use `./test/initial_setup.sh`
+To **setup** the tests environment use `./test/initial_setup.sh`
 
 The script will create a .env file in the test folder.
 When the script is complete you only need to add a `ADMIN_SECRET_KEY` and a `TEST_USER_SECRET` to the `.env`.
@@ -36,12 +161,13 @@ SUI_NODE=
 GET_WORKER_TIMEOUT_MS=1000
 ```
 
-_Tip: You can see your addresses' secret keys by running `cat ~/.sui/sui_config/sui.keystore`_
+_Tip: You can see your addresses' secret keys by running `cat ~/.sui/sui_config/sui.keystore`. Each
+secret's corresponding address is in the same row line that appears in `sui client addresses`_.
 
 We use the [jest](https://jestjs.io/) framework for testing. Having installed the project's packages with `npm install`, you can run the tests by either:
 
 1. The vscode `jest` extension (Extension Id: **Orta.vscode-jest**) - [Recommended]
 
 The extension provides a flask to the IDE sidebar where you run the tests (altogether or one-by-one) and show the results in the editor. You can also run the tests in debug mode and set breakpoints in the code. Very useful when doing [TDD](https://en.wikipedia.org/wiki/Test-driven_development).
 
-2. ... or from the command line using `node_modules/.bin/jest --verbose` - Best for CI/CD
+2. ... or from the command line using `npm run test` - Best for CI/CD

src/executorServiceHandler.ts

@@ -1,8 +1,8 @@
 import { SuiClient } from '@mysten/sui.js/client';
 import { Keypair } from '@mysten/sui.js/src/cryptography';
 import { TransactionBlock } from '@mysten/sui.js/transactions';
-
 import { Pool, SplitStrategy } from './pool';
+import { getEnvironmentVariables } from './helpers';
 
 type WorkerPool = {
   status: 'available' | 'busy';
@@ -12,11 +12,13 @@ type WorkerPool = {
 export class ExecutorServiceHandler {
   private _mainPool: Pool;
   private _workers: WorkerPool[] = [];
+  private readonly _getWorkerTimeoutMs: number;
   private constructor(mainPool: Pool) {
+    this._getWorkerTimeoutMs = getEnvironmentVariables().GET_WORKER_TIMEOUT_MS;
     this._mainPool = mainPool;
   }
 
-  public static initialize(keypair: Keypair, client: SuiClient) {
+  public static async initialize(keypair: Keypair, client: SuiClient) {
     return Pool.full({ keypair: keypair, client }).then((pool) => {
       return new ExecutorServiceHandler(pool);
     });
@@ -30,13 +32,15 @@ export class ExecutorServiceHandler {
   ) {
     let res;
     do {
-      res = await this.executeFlow(txb, client, splitStrategy);
+      try {
+        res = await this.executeFlow(txb, client, splitStrategy);
+      } catch (e) {
+        console.log('Error executing transaction block')
+        console.log(e);
+        continue;
+      }
       if (res) {
         return res;
-      } else {
-        console.log(
-          `Failed to execute the txb - [remaining retries: ${retries}]`,
-        );
       }
     } while (retries-- > 0);
     throw new Error(
@@ -79,10 +83,7 @@ export class ExecutorServiceHandler {
   If an available worker is not found in the time span of TIMEOUT_MS, return undefined.
   */
   private getAWorker(): WorkerPool | undefined {
-    if (!process.env.GET_WORKER_TIMEOUT_MS) {
-      throw new Error("Environment variable 'GET_WORKER_TIMEOUT_MS' not set.");
-    }
-    const timeoutMs = parseInt(process.env.GET_WORKER_TIMEOUT_MS);
+    const timeoutMs = this._getWorkerTimeoutMs;
     const startTime = new Date().getTime();
     while (new Date().getTime() - startTime < timeoutMs) {
       const result = this._workers.find(
@@ -109,7 +110,7 @@ export class ExecutorServiceHandler {
     that is produced is added to the workers array.
    */
   private addWorker(splitStrategy?: SplitStrategy) {
-    console.log("Splitting main pool to add new worker Pool...");
+    console.log('Splitting main pool to add new worker Pool...');
     const newPool = this._mainPool.split(splitStrategy);
     this._workers.push({ status: 'available', pool: newPool });
   }

src/helpers.ts

@@ -20,6 +20,44 @@ export function getKeyPair(privateKey: string): Ed25519Keypair {
   return Ed25519Keypair.fromSecretKey(Uint8Array.from(privateKeyArray));
 }
 
+type EnvironmentVariables = {
+  NFT_APP_PACKAGE_ID: string;
+  NFT_APP_ADMIN_CAP: string;
+  SUI_NODE: string;
+  ADMIN_ADDRESS: string;
+  ADMIN_SECRET_KEY: string;
+  TEST_USER_ADDRESS: string;
+  TEST_USER_SECRET: string;
+  GET_WORKER_TIMEOUT_MS: number;
+};
+
+export function getEnvironmentVariables() {
+  const env = {
+    NFT_APP_PACKAGE_ID: process.env.NFT_APP_PACKAGE_ID ?? '',
+    NFT_APP_ADMIN_CAP: process.env.NFT_APP_ADMIN_CAP ?? '',
+    SUI_NODE: process.env.SUI_NODE ?? '',
+    ADMIN_ADDRESS: process.env.ADMIN_ADDRESS ?? '',
+    ADMIN_SECRET_KEY: process.env.ADMIN_SECRET_KEY ?? '',
+    TEST_USER_ADDRESS: process.env.TEST_USER_ADDRESS ?? '',
+    TEST_USER_SECRET: process.env.TEST_USER_SECRET ?? '',
+    GET_WORKER_TIMEOUT_MS: parseInt(
+      process.env.GET_WORKER_TIMEOUT_MS ?? '10000',
+    ),
+  } as EnvironmentVariables;
+
+  checkForMissingVariables(env);
+
+  return env;
+}
+
+function checkForMissingVariables(env: EnvironmentVariables) {
+  for (const [key, value] of Object.entries(env)) {
+    if (!value) {
+      throw new Error(`Missing environment variable ${key}`);
+    }
+  }
+}
+
 export async function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -41,31 +79,29 @@ export function compareMaps<T>(map1: Map<string, T>, map2: Map<string, T>) {
 
 export class SetupTestsHelper {
   public MINIMUM_COIN_BALANCE: number;
-
+  private readonly env: EnvironmentVariables;
   private client: SuiClient;
   private adminKeypair: Ed25519Keypair;
 
   public readonly objects: SuiObjectResponse[] = [];
   private suiCoins: SuiObjectResponse[] = [];
 
   constructor() {
-    this.MINIMUM_COIN_BALANCE = 700000000
-    if (!process.env.SUI_NODE) {
-      throw new Error('SUI_NODE env variable is not set');
-    }
+    this.env = getEnvironmentVariables();
+    this.MINIMUM_COIN_BALANCE = 700000000;
     this.client = new SuiClient({
-      url: process.env.SUI_NODE,
+      url: this.env.SUI_NODE,
     });
-    if (!process.env.ADMIN_SECRET_KEY) {
-      throw new Error('ADMIN_SECRET_KEY env variable is not set');
-    }
-    this.adminKeypair = getKeyPair(process.env.ADMIN_SECRET_KEY);
+    this.adminKeypair = getKeyPair(this.env.ADMIN_SECRET_KEY);
   }
 
   /*
   Reassure that the admin has enough coins and objects to run the tests
    */
-  public async setupAdmin(minimumObjectsNeeded: number, minimumCoinsNeeded: number) {
+  public async setupAdmin(
+    minimumObjectsNeeded: number,
+    minimumCoinsNeeded: number,
+  ) {
     const setup = async () => {
       await this.parseCurrentCoinsAndObjects();
       await this.assureAdminHasEnoughObjects(minimumObjectsNeeded);
@@ -138,18 +174,15 @@ export class SetupTestsHelper {
 
   private async addNewObjectToAccount() {
     const mintAndTransferTxb = new TransactionBlock();
-    if (!process.env.NFT_APP_ADMIN_CAP) {
-      throw new Error('NFT_APP_ADMIN_CAP env variable is not set');
-    }
     const hero = mintAndTransferTxb.moveCall({
       arguments: [
-        mintAndTransferTxb.object(process.env.NFT_APP_ADMIN_CAP),
+        mintAndTransferTxb.object(this.env.NFT_APP_ADMIN_CAP),
         mintAndTransferTxb.pure('zed'),
         mintAndTransferTxb.pure('gold'),
         mintAndTransferTxb.pure(3),
         mintAndTransferTxb.pure('ipfs://example.com/'),
       ],
-      target: `${process.env.NFT_APP_PACKAGE_ID}::hero_nft::mint_hero`,
+      target: `${this.env.NFT_APP_PACKAGE_ID}::hero_nft::mint_hero`,
     });
     // Transfer to self
     mintAndTransferTxb.transferObjects(
@@ -175,13 +208,14 @@ export class SetupTestsHelper {
   private async addNewCoinToAccount(cointToSplit: string) {
     const txb = new TransactionBlock();
     const coinToPay = await this.client.getObject({ id: cointToSplit });
-    const newcoins1 = txb.splitCoins(txb.gas, [txb.pure(this.MINIMUM_COIN_BALANCE)]);
-    const newcoins2 = txb.splitCoins(txb.gas, [txb.pure(this.MINIMUM_COIN_BALANCE)]);
+    const newcoins1 = txb.splitCoins(txb.gas, [
+      txb.pure(this.MINIMUM_COIN_BALANCE),
+    ]);
+    const newcoins2 = txb.splitCoins(txb.gas, [
+      txb.pure(this.MINIMUM_COIN_BALANCE),
+    ]);
     txb.transferObjects(
-      [
-        newcoins1,
-        newcoins2
-      ],
+      [newcoins1, newcoins2],
       txb.pure(this.adminKeypair.toSuiAddress()),
     );
     txb.setGasBudget(100000000);
@@ -199,11 +233,15 @@ export class SetupTestsHelper {
       .then((txRes) => {
         const status = txRes.effects?.status?.status;
         if (status !== 'success') {
-          throw new Error(`Failed to split and add new coin to admin account! ${status}`)
+          throw new Error(
+            `Failed to split and add new coin to admin account! ${status}`,
+          );
         }
       })
       .catch((err) => {
-        throw new Error(`Failed to split coin <${cointToSplit}> and add new coin to admin account! ${err}`)
+        throw new Error(
+          `Failed to split coin <${cointToSplit}> and add new coin to admin account! ${err}`,
+        );
       });
   }