fix: multichain e2e rewards available condition is flakey (#9816)

refs: #9815 ## Description - allow `retryUntilCondition` callers to override default `RetryOptions = { maxRetries?: number; retryIntervalMs?: number; }` parameter - increase attempts and retry interval for "staking rewards available" condition in `stake-ica.test.ts`. ### Security Considerations n/a ### Scaling Considerations n/a ### Documentation Considerations n/a ### Testing Considerations The goal of this PR is reduce observed CI flakes. ### Upgrade Considerations n/a
Agoric · Jul 31, 2024 · 20cb52f · 20cb52f
2 parents 8718c07 + ad8b64f
commit 20cb52f
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 25 deletions.
diff --git a/multichain-testing/test/stake-ica.test.ts b/multichain-testing/test/stake-ica.test.ts
@@ -179,6 +179,10 @@ const stakeScenario = test.macro(async (t, scenario: StakeIcaScenario) => {
       return Number(total?.[0]?.amount) > 0;
     },
     `rewards available on ${scenario.chain}`,
+    {
+      retryIntervalMs: 5000,
+      maxRetries: 8,
+    },
   );
   t.log('reward:', total[0]);
   t.log('WithrawReward offer from continuing inv');
@@ -242,6 +246,7 @@ const stakeScenario = test.macro(async (t, scenario: StakeIcaScenario) => {
   t.log('Current Balance:', currentBalances[0]);
 
   console.log('waiting for unbonding period');
+  // XXX reference `120000` from chain state + `maxClockSkew`
   await sleep(120000);
   const { balances: rewardsWithUndelegations } = await retryUntilCondition(
     () => queryClient.queryBalances(address),

diff --git a/multichain-testing/test/support.ts b/multichain-testing/test/support.ts
@@ -55,7 +55,10 @@ export const commonSetup = async (t: ExecutionContext) => {
   const tools = await makeAgdTools(t.log, childProcess);
   const keyring = await makeKeyring(tools);
   const deployBuilder = makeDeployBuilder(tools, fse.readJSON, execa);
-  const retryUntilCondition = makeRetryUntilCondition(t.log);
+  const retryUntilCondition = makeRetryUntilCondition({
+    log: t.log,
+    setTimeout: globalThis.setTimeout,
+  });
 
   return { useChain, ...tools, ...keyring, retryUntilCondition, deployBuilder };
 };

diff --git a/multichain-testing/test/tools/wallet.test.ts b/multichain-testing/test/tools/wallet.test.ts
@@ -2,13 +2,12 @@ import anyTest from '@endo/ses-ava/prepare-endo.js';
 import type { TestFn } from 'ava';
 import { makeQueryClient } from '../../tools/query.js';
 import { createWallet } from '../../tools/wallet.js';
-import { sleep } from '../../tools/sleep.js';
 import { commonSetup } from '../support.js';
 
 const test = anyTest as TestFn<Record<string, never>>;
 
 const walletScenario = test.macro(async (t, scenario: string) => {
-  const { useChain } = await commonSetup(t);
+  const { useChain, retryUntilCondition } = await commonSetup(t);
 
   const prefix = useChain(scenario).chain.bech32_prefix;
   const wallet = await createWallet(prefix);
@@ -30,9 +29,12 @@ const walletScenario = test.macro(async (t, scenario: string) => {
   await creditFromFaucet(addr);
   // XXX needed to avoid race condition between faucet POST and LCD Query
   // see https://github.com/cosmology-tech/starship/issues/417
-  await sleep(1000, t.log);
+  const { balances: updatedBalances } = await retryUntilCondition(
+    () => queryClient.queryBalances(addr),
+    ({ balances }) => !!balances.length,
+    `${scenario} balance available from faucet`,
+  );
 
-  const { balances: updatedBalances } = await queryClient.queryBalances(addr);
   const expectedDenom = scenario === 'osmosis' ? 'uosmo' : 'uatom';
   t.like(updatedBalances, [{ denom: expectedDenom, amount: '10000000000' }]);
   t.log('Updated balances:', updatedBalances);

diff --git a/multichain-testing/tools/sleep.ts b/multichain-testing/tools/sleep.ts
@@ -1,18 +1,36 @@
+const ambientSetTimeout = globalThis.setTimeout;
+
 type Log = (...values: unknown[]) => void;
 
-export const sleep = (ms: number, log: Log = () => {}) =>
+type SleepOptions = {
+  log?: Log;
+  setTimeout?: typeof ambientSetTimeout;
+};
+
+export const sleep = (
+  ms: number,
+  { log = () => {}, setTimeout = ambientSetTimeout }: SleepOptions = {},
+) =>
   new Promise(resolve => {
     log(`Sleeping for ${ms}ms...`);
     setTimeout(resolve, ms);
   });
 
+type RetryOptions = {
+  maxRetries?: number;
+  retryIntervalMs?: number;
+} & SleepOptions;
+
 const retryUntilCondition = async <T>(
   operation: () => Promise<T>,
   condition: (result: T) => boolean,
   message: string,
-  maxRetries: number,
-  retryIntervalMs: number,
-  log: Log,
+  {
+    maxRetries = 6,
+    retryIntervalMs = 3500,
+    log = () => {},
+    setTimeout = ambientSetTimeout,
+  }: RetryOptions = {},
 ): Promise<T> => {
   console.log({ maxRetries, retryIntervalMs, message });
   let retries = 0;
@@ -35,28 +53,25 @@ const retryUntilCondition = async <T>(
     console.log(
       `Retry ${retries}/${maxRetries} - Waiting for ${retryIntervalMs}ms for ${message}...`,
     );
-    await sleep(retryIntervalMs, log);
+    await sleep(retryIntervalMs, { log, setTimeout });
   }
 
   throw new Error(`${message} condition failed after ${maxRetries} retries.`);
 };
 
-export const makeRetryUntilCondition =
-  (
-    log: Log = () => {},
-    maxRetries: number = 6,
-    retryIntervalMs: number = 3500,
-  ) =>
-  <T>(
+export const makeRetryUntilCondition = (defaultOptions: RetryOptions = {}) => {
+  /**
+   * Retry an asynchronous operation until a condition is met.
+   * Defaults to maxRetries = 6, retryIntervalMs = 3500
+   */
+  return <T>(
     operation: () => Promise<T>,
     condition: (result: T) => boolean,
     message: string,
+    options?: RetryOptions,
   ) =>
-    retryUntilCondition(
-      operation,
-      condition,
-      message,
-      maxRetries,
-      retryIntervalMs,
-      log,
-    );
+    retryUntilCondition(operation, condition, message, {
+      ...defaultOptions,
+      ...options,
+    });
+};