Merge pull request #154 from Cognigy/bug/38102-fix-rate-limits-clone-…

…agent

fix(bug) fixes and improvements on clone agent command

README.md

@@ -83,6 +83,8 @@ You can force these checks to be skipped with the `-y` flag.
 
 Example: `cognigy clone -y` or `cognigy clone --forceYes`
 
+**Some environments may have rate limits. This means that when executing commands that require a high number of API requests (e.g. clone), the env rate limit may be reached, and the process stops by throwing an error.**
+
 ### Command: init
 
 `cognigy init`

src/commands/clone.ts

@@ -40,11 +40,9 @@ export const clone = async ({ resourceType = 'agent', forceYes = false }): Promi
 
     switch (resourceType) {
         case "agent":
-            await Promise.all([
-                cloneFlows(33),
-                cloneEndpoints(33),
-                cloneLexicons(33)
-            ]);
+            await cloneFlows(33);
+            await cloneEndpoints(33);
+            await cloneLexicons(33);
             break;
 
         case "flows":

src/lib/flows.ts

@@ -22,6 +22,7 @@ import { indexAll } from '../utils/indexAll';
 import { ILocaleIndexItem_2_0 } from '@cognigy/rest-api-client/build/shared/interfaces/restAPI/resources/locales/v2.0';
 import { IIntent } from '@cognigy/rest-api-client/build/shared/interfaces/resources/intent/IIntent';
 import { IIndexFlowsRestReturnValue_2_0 } from '@cognigy/rest-api-client';
+import { chunkArray } from '../utils/chunk';
 
 /**
  * Clones Cognigy Flows to disk
@@ -34,18 +35,22 @@ export const cloneFlows = async (availableProgress: number): Promise<void> => {
 
     // query Cognigy.AI for all Flows in this agent
     const flows = await indexAll(CognigyClient.indexFlows)({
-        "projectId": CONFIG.agent
+        projectId: CONFIG.agent,
     });
 
     const progressPerFlow = availableProgress / flows.items.length;
 
-    // create a sub-folder, chart.json and config.json for each Flow
-    const flowsPromiseArr = []
+    const flowsPromiseArr: Array<() => Promise<void>> = [];
     for (let flow of flows.items) {
-        flowsPromiseArr.push(pullFlow(flow.name, progressPerFlow));
+        flowsPromiseArr.push(() => pullFlow(flow.name, progressPerFlow));
     }
 
-    await Promise.all(flowsPromiseArr);
+    // create chunks of x pullFlow functions and promise all, in order to speed up the process
+    const chunkedFlowsPromiseArr = chunkArray(flowsPromiseArr, 5);
+
+    for (let chunk of chunkedFlowsPromiseArr) {
+        await Promise.all(chunk.map((func) => func()));
+    }
 
     return Promise.resolve();
 };

src/lib/lexicons.ts

@@ -13,6 +13,7 @@ import CognigyClient from '../utils/cognigyClient';
 import { makeAxiosRequest } from '../utils/axiosClient';
 import { checkCreateDir, checkTask, removeCreateDir } from '../utils/checks';
 import { indexAll } from '../utils/indexAll';
+import { chunkArray } from '../utils/chunk';
 
 /**
  * Clones Cognigy Lexicons to disk
@@ -36,9 +37,15 @@ export const cloneLexicons = async (availableProgress: number): Promise<void> =>
 
     const incrementPerLexicon = availableProgress / lexicons.items.length;
 
-    // create a sub-folder, chart.json and config.json for each Flow
+    const lexiconsPromiseArr: Array<() => Promise<void>> = [];
     for (let lexicon of lexicons.items) {
-        await pullLexicon(lexicon.name, incrementPerLexicon);
+        lexiconsPromiseArr.push(() => pullLexicon(lexicon.name, incrementPerLexicon));
+    }
+
+    const chunkedLexiconssPromiseArr = chunkArray(lexiconsPromiseArr, 5);
+
+    for (let chunk of chunkedLexiconssPromiseArr) {
+        await Promise.all(chunk.map((func) => func()));
     }
 
     return Promise.resolve();

src/utils/chunk.ts

@@ -0,0 +1,8 @@
+export const chunkArray = <T>(inputArray: T[], size: number): T[][] => {
+    return inputArray.reduce((resultArray, _, i) => {
+        if (i % size === 0) {
+            resultArray.push(inputArray.slice(i, i + size));
+        }
+        return resultArray;
+    }, []);
+};