merge with master

package.json

@@ -58,7 +58,7 @@
     "mocha": "^10.4.0",
     "prettier": "^3.2.5",
     "rimraf": "^5.0.5",
-    "sinon": "^17.0.1",
+    "sinon": "^17.0.2",
     "ts-node": "^10.9.2",
     "typescript": "^5.4.5"
   },

src/constants.ts

@@ -4,3 +4,6 @@ export const JRPC_METHODS = {
   IMPORT_SHARES: "ImportShares",
   GET_SHARE_OR_KEY_ASSIGN: "GetShareOrKeyAssign",
 };
+
+export const SAPPHIRE_METADATA_URL = "https://node-1.node.web3auth.io/metadata";
+export const SAPPHIRE_DEVNET_METADATA_URL = "https://node-1.dev-node.web3auth.io/metadata";

src/helpers/common.ts

@@ -1,9 +1,10 @@
+import { JRPCResponse } from "@toruslabs/constants";
 import { Ecies } from "@toruslabs/eccrypto";
 import { BN } from "bn.js";
 import { ec as EC } from "elliptic";
 import JsonStringify from "json-stable-stringify";
 
-import { EciesHex, KeyType, VerifierLookupResponse } from "../interfaces";
+import { CommitmentRequestResult, EciesHex, KeyType, VerifierLookupResponse } from "../interfaces";
 import { keccak256 } from "./keyUtils";
 
 export const ed25519Curve = new EC("ed25519");
@@ -125,3 +126,60 @@ export function calculateMedian(arr: number[]): number {
   const mid2 = sortedArr[arrSize / 2];
   return (mid1 + mid2) / 2;
 }
+
+export function waitFor(milliseconds: number) {
+  return new Promise((resolve, reject) => {
+    // hack to bypass eslint warning.
+    if (milliseconds > 0) {
+      setTimeout(resolve, milliseconds);
+    } else {
+      reject(new Error("value of milliseconds must be greater than 0"));
+    }
+  });
+}
+
+export function retryCommitment(executionPromise: () => Promise<JRPCResponse<CommitmentRequestResult>>, maxRetries: number) {
+  // Notice that we declare an inner function here
+  // so we can encapsulate the retries and don't expose
+  // it to the caller. This is also a recursive function
+  async function retryWithBackoff(retries: number) {
+    try {
+      // we don't wait on the first attempt
+      if (retries > 0) {
+        // on every retry, we exponentially increase the time to wait.
+        // Here is how it looks for a `maxRetries` = 4
+        // (2 ** 1) * 100 = 200 ms
+        // (2 ** 2) * 100 = 400 ms
+        // (2 ** 3) * 100 = 800 ms
+        const timeToWait = 2 ** retries * 100;
+        await waitFor(timeToWait);
+      }
+      const a = await executionPromise();
+      return a;
+    } catch (e: unknown) {
+      const errorMsg = (e as Error).message;
+      const acceptedErrorMsgs = [
+        // Slow node
+        "Timed out",
+        "Failed to fetch",
+        "fetch failed",
+        "Load failed",
+        "cancelled",
+        "NetworkError when attempting to fetch resource.",
+        // Happens when the node is not reachable (dns issue etc)
+        "TypeError: Failed to fetch", // All except iOS and Firefox
+        "TypeError: cancelled", // iOS
+        "TypeError: NetworkError when attempting to fetch resource.", // Firefox
+      ];
+
+      if (retries < maxRetries && (acceptedErrorMsgs.includes(errorMsg) || (errorMsg && errorMsg.includes("reason: getaddrinfo EAI_AGAIN")))) {
+        // only retry if we didn't reach the limit
+        // otherwise, let the caller handle the error
+        return retryWithBackoff(retries + 1);
+      }
+      throw e;
+    }
+  }
+
+  return retryWithBackoff(0);
+}

src/helpers/metadataUtils.ts

@@ -1,11 +1,22 @@
+import { LEGACY_NETWORKS_ROUTE_MAP, TORUS_LEGACY_NETWORK_TYPE, TORUS_NETWORK_TYPE, TORUS_SAPPHIRE_NETWORK } from "@toruslabs/constants";
 import { decrypt } from "@toruslabs/eccrypto";
 import { Data, post } from "@toruslabs/http-helpers";
 import BN from "bn.js";
-import { ec } from "elliptic";
+import { ec as EC } from "elliptic";
 import stringify from "json-stable-stringify";
 import log from "loglevel";
 
-import { EciesHex, EncryptedSeed, GetOrSetNonceResult, KeyType, MetadataParams, NonceMetadataParams, SetNonceData } from "../interfaces";
+import { SAPPHIRE_DEVNET_METADATA_URL, SAPPHIRE_METADATA_URL } from "../constants";
+import {
+  EciesHex,
+  EncryptedSeed,
+  GetOrSetNonceResult,
+  KeyType,
+  MetadataParams,
+  NonceMetadataParams,
+  SapphireMetadataParams,
+  SetNonceData,
+} from "../interfaces";
 import { encParamsHexToBuf, secp256k1Curve } from "./common";
 import { getSecpKeyFromEd25519, keccak256 } from "./keyUtils";
 
@@ -42,7 +53,7 @@ export async function decryptNodeDataWithPadding(eciesData: EciesHex, ciphertext
   }
 }
 
-export function generateMetadataParams(ecCurve: ec, serverTimeOffset: number, message: string, privateKey: BN): MetadataParams {
+export function generateMetadataParams(ecCurve: EC, serverTimeOffset: number, message: string, privateKey: BN): MetadataParams {
   const key = ecCurve.keyFromPrivate(privateKey.toString("hex", 64));
   const setData = {
     data: message,
@@ -111,7 +122,7 @@ export function generateNonceMetadataParams(
 
 export async function getOrSetNonce(
   metadataHost: string,
-  ecCurve: ec,
+  ecCurve: EC,
   serverTimeOffset: number,
   X: string,
   Y: string,
@@ -164,7 +175,7 @@ export async function getOrSetNonce(
 }
 export async function getNonce(
   legacyMetadataHost: string,
-  ecCurve: ec,
+  ecCurve: EC,
   serverTimeOffset: number,
   X: string,
   Y: string,
@@ -191,3 +202,38 @@ export const decryptSeedData = async (seedBase64: string, finalUserKey: BN) => {
 
   return decText;
 };
+export async function getOrSetSapphireMetadataNonce(
+  network: TORUS_NETWORK_TYPE,
+  X: string,
+  Y: string,
+  serverTimeOffset?: number,
+  privKey?: BN
+): Promise<GetOrSetNonceResult> {
+  if (LEGACY_NETWORKS_ROUTE_MAP[network as TORUS_LEGACY_NETWORK_TYPE]) {
+    throw new Error("getOrSetSapphireMetadataNonce should only be used for sapphire networks");
+  }
+  let data: SapphireMetadataParams = {
+    pub_key_X: X,
+    pub_key_Y: Y,
+    key_type: "secp256k1",
+    set_data: { operation: "getOrSetNonce" },
+  };
+  if (privKey) {
+    const key = secp256k1Curve.keyFromPrivate(privKey.toString("hex", 64));
+
+    const setData = {
+      operation: "getOrSetNonce",
+      timestamp: new BN(~~(serverTimeOffset + Date.now() / 1000)).toString(16),
+    };
+    const sig = key.sign(keccak256(Buffer.from(stringify(setData), "utf8")).slice(2));
+    data = {
+      ...data,
+      set_data: setData,
+      signature: Buffer.from(sig.r.toString(16, 64) + sig.s.toString(16, 64) + new BN("").toString(16, 2), "hex").toString("base64"),
+    };
+  }
+
+  const metadataUrl = network === TORUS_SAPPHIRE_NETWORK.SAPPHIRE_DEVNET ? SAPPHIRE_DEVNET_METADATA_URL : SAPPHIRE_METADATA_URL;
+
+  return post<GetOrSetNonceResult>(`${metadataUrl}/get_or_set_nonce`, data, undefined, { useAPIKey: true });
+}

src/helpers/nodeUtils.ts

@@ -26,7 +26,7 @@ import {
 } from "../interfaces";
 import log from "../loglevel";
 import { Some } from "../some";
-import { calculateMedian, getProxyCoordinatorEndpointIndex, kCombinations, normalizeKeysResult, thresholdSame } from "./common";
+import { calculateMedian, getProxyCoordinatorEndpointIndex, kCombinations, normalizeKeysResult, retryCommitment, thresholdSame } from "./common";
 import {
   derivePubKey,
   generateAddressFromPrivKey,
@@ -37,7 +37,14 @@ import {
   keccak256,
 } from "./keyUtils";
 import { lagrangeInterpolation } from "./langrangeInterpolatePoly";
-import { decryptNodeData, decryptNodeDataWithPadding, decryptSeedData, getMetadata, getOrSetNonce } from "./metadataUtils";
+import {
+  decryptNodeData,
+  decryptNodeDataWithPadding,
+  decryptSeedData,
+  getMetadata,
+  getOrSetNonce,
+  getOrSetSapphireMetadataNonce,
+} from "./metadataUtils";
 
 export const GetPubKeyOrKeyAssign = async (params: {
   endpoints: string[];
@@ -69,7 +76,7 @@ export const GetPubKeyOrKeyAssign = async (params: {
 
   let nonceResult: GetOrSetNonceResult | undefined;
   const nodeIndexes: number[] = [];
-  const result = await Some<void | JRPCResponse<VerifierLookupResponse>, KeyLookupResult>(lookupPromises, (lookupResults) => {
+  const result = await Some<void | JRPCResponse<VerifierLookupResponse>, KeyLookupResult>(lookupPromises, async (lookupResults) => {
     const lookupPubKeys = lookupResults.filter((x1) => {
       if (x1 && !x1.error) {
         return x1;
@@ -101,6 +108,18 @@ export const GetPubKeyOrKeyAssign = async (params: {
           }
         }
       }
+
+      // if nonce result is not returned by nodes, fetch directly from metadata
+      if (!nonceResult) {
+        const metadataNonceResult = await getOrSetSapphireMetadataNonce(network, keyResult.keys[0].pub_key_X, keyResult.keys[0].pub_key_Y);
+        // rechecking nonceResult to avoid promise race condition.
+        if (!nonceResult && metadataNonceResult) {
+          nonceResult = metadataNonceResult;
+          if (nonceResult.nonce) {
+            delete nonceResult.nonce;
+          }
+        }
+      }
     }
 
     const serverTimeOffsets: number[] = [];
@@ -144,7 +163,7 @@ export async function retrieveOrImportShare(params: {
   ecCurve: ec;
   keyType: KeyType;
   allowHost: string;
-  network: string;
+  network: TORUS_NETWORK_TYPE;
   clientId: string;
   endpoints: string[];
   indexes: number[];
@@ -223,25 +242,25 @@ export async function retrieveOrImportShare(params: {
         VerifierIdentifier string `json:"verifieridentifier"`
       } 
       */
-    const p = post<JRPCResponse<CommitmentRequestResult>>(
-      endpoints[i],
-      generateJsonRPCObject(JRPC_METHODS.COMMITMENT_REQUEST, {
-        messageprefix: "mug00",
-        keytype: keyType,
-        tokencommitment: tokenCommitment.slice(2),
-        temppubx: pubKeyX,
-        temppuby: pubKeyY,
-        verifieridentifier: verifier,
-        verifier_id: verifierParams.verifier_id,
-        extended_verifier_id: verifierParams.extended_verifier_id,
-        is_import_key_flow: true,
-      }),
-      null,
-      { logTracingHeader: config.logRequestTracing }
-    ).catch((err) => {
-      log.error("commitment error", err);
-    });
-    promiseArr.push(p);
+    const p = () =>
+      post<JRPCResponse<CommitmentRequestResult>>(
+        endpoints[i],
+        generateJsonRPCObject(JRPC_METHODS.COMMITMENT_REQUEST, {
+          messageprefix: "mug00",
+          keytype: keyType,
+          tokencommitment: tokenCommitment.slice(2),
+          temppubx: pubKeyX,
+          temppuby: pubKeyY,
+          verifieridentifier: verifier,
+          verifier_id: verifierParams.verifier_id,
+          extended_verifier_id: verifierParams.extended_verifier_id,
+          is_import_key_flow: true,
+        }),
+        null,
+        { logTracingHeader: config.logRequestTracing }
+      );
+    const r = retryCommitment(p, 4);
+    promiseArr.push(r);
   }
   // send share request once k + t number of commitment requests have completed
   return Some<void | JRPCResponse<CommitmentRequestResult>, (void | JRPCResponse<CommitmentRequestResult>)[]>(promiseArr, (resultArr) => {
@@ -314,7 +333,7 @@ export async function retrieveOrImportShare(params: {
       }
     }
 
-    return Promise.reject(new Error(`invalid commitment results, ${JSON.stringify(resultArr)}`));
+    return Promise.reject(new Error(`invalid commitment results ${JSON.stringify(resultArr)}`));
   })
     .then((responses) => {
       const promiseArrRequest: Promise<void | JRPCResponse<ShareRequestResult> | JRPCResponse<ShareRequestResult[]>>[] = [];
@@ -405,7 +424,7 @@ export async function retrieveOrImportShare(params: {
             }),
             null,
             { logTracingHeader: config.logRequestTracing }
-          ).catch((err) => log.error("share req", err));
+          );
           promiseArrRequest.push(p);
         }
       }
@@ -473,24 +492,11 @@ export async function retrieveOrImportShare(params: {
           }
         });
 
-        // if both thresholdNonceData and extended_verifier_id are not available
-        // then we need to throw other wise address would be incorrect.
-        if (!thresholdNonceData && !verifierParams.extended_verifier_id && !LEGACY_NETWORKS_ROUTE_MAP[network as TORUS_LEGACY_NETWORK_TYPE]) {
-          throw new Error(
-            `invalid metadata result from nodes, nonce metadata is empty for verifier: ${verifier} and verifierId: ${verifierParams.verifier_id}`
-          );
-        }
-
-        const thresholdReqCount = ~~(endpoints.length / 2) + 1;
+        const thresholdReqCount = canImportedShares ? endpoints.length : ~~(endpoints.length / 2) + 1;
         // optimistically run lagrange interpolation once threshold number of shares have been received
         // this is matched against the user public key to ensure that shares are consistent
         // Note: no need of thresholdMetadataNonce for extended_verifier_id key
-
-        if (
-          completedRequests.length >= thresholdReqCount &&
-          thresholdPublicKey &&
-          (thresholdNonceData || verifierParams.extended_verifier_id || LEGACY_NETWORKS_ROUTE_MAP[network as TORUS_LEGACY_NETWORK_TYPE])
-        ) {
+        if (completedRequests.length >= thresholdReqCount && thresholdPublicKey) {
           const sharePromises: Promise<void | Buffer>[] = [];
           const sessionTokenSigPromises: Promise<void | Buffer>[] = [];
           const sessionTokenPromises: Promise<void | Buffer>[] = [];
@@ -661,11 +667,25 @@ export async function retrieveOrImportShare(params: {
       const { privateKey, sessionTokenData, thresholdNonceData, nodeIndexes, isNewKey, serverTimeOffsetResponse, thresholdPubKey } = res;
       let nonceResult = thresholdNonceData;
       if (!privateKey) throw new Error("Invalid private key returned");
+
       const oAuthKey = privateKey;
       const oAuthPubKey = derivePubKey(ecCurve, oAuthKey);
       const oAuthPubkeyX = oAuthPubKey.getX().toString("hex", 64);
       const oAuthPubkeyY = oAuthPubKey.getY().toString("hex", 64);
 
+      // if both thresholdNonceData and extended_verifier_id are not available
+      // then we need to throw other wise address would be incorrect.
+      if (!nonceResult && !verifierParams.extended_verifier_id && !LEGACY_NETWORKS_ROUTE_MAP[network as TORUS_LEGACY_NETWORK_TYPE]) {
+        const metadataNonceResult = await getOrSetSapphireMetadataNonce(network, oAuthPubkeyX, oAuthPubkeyY, serverTimeOffset, oAuthKey);
+        // rechecking nonceResult to avoid promise race condition.
+        if (metadataNonceResult && !thresholdNonceData) {
+          nonceResult = metadataNonceResult;
+        } else {
+          throw new Error(
+            `invalid metadata result from nodes, nonce metadata is empty for verifier: ${verifier} and verifierId: ${verifierParams.verifier_id}`
+          );
+        }
+      }
       let metadataNonce = new BN(nonceResult?.nonce ? nonceResult.nonce.padStart(64, "0") : "0", "hex");
       let finalPubKey: curve.base.BasePoint;
       let pubNonce: { X: string; Y: string } | undefined;