MultipleLlmExecutionTools

webgptorg · May 25, 2024 · 4bdf112 · 4bdf112
1 parent dc597eb
commit 4bdf112
Show file tree

Hide file tree

Showing 11 changed files with 372 additions and 115 deletions.
diff --git a/.vscode/terminals.json b/.vscode/terminals.json
@@ -131,6 +131,12 @@
             "onlySingle": true,
             "focus": true
         },
+        {
+            "name": "🧸 Multiple LLMs Playground",
+            "command": "ts-node ./src/execution/plugins/llm-execution-tools/multiple/playground/playground.ts",
+            "onlySingle": true,
+            "focus": true
+        },
         {
             "name": "🔼👑 Release major version",
             "command": "npm version major",

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -288,6 +288,7 @@ Add new OpenaAI models `gpt-4o` and `gpt-4o-2024-05-13`
 
 Add support for Claude \\ Anthropic models via package `@promptbook/anthropic-claude` and add Azure OpenAI models via package `@promptbook/azure-openai`
 
+-   Export `MultipleLlmExecutionTools` from `@promptbook/core`
 -   Always use "modelName" not just "model"
 -   Standartization of model providers
 

diff --git a/README.md b/README.md
@@ -471,9 +471,10 @@ Internally it calls OpenAI, Azure, GPU, proxy, cache, logging,...
 -   _(Not implemented yet)_ `BardExecutionTools`
 -   _(Not implemented yet)_ `LamaExecutionTools`
 -   _(Not implemented yet)_ `GpuExecutionTools`
--   And a special case are `RemoteLlmExecutionTools` that connect to a remote server and run one of the above execution tools on that server.
--   The second special case is `MockedEchoLlmExecutionTools` that is used for testing and mocking.
--   The third special case is `LogLlmExecutionToolsWrapper` that is technically also an execution tools but it is more proxy wrapper around other execution tools that logs all calls to execution tools.
+-   And a special case are `MultipleLlmExecutionTools` that combines multiple execution tools together and tries to execute the prompt on the best one.
+-   Another special case are `RemoteLlmExecutionTools` that connect to a remote server and run one of the above execution tools on that server.
+-   The another special case is `MockedEchoLlmExecutionTools` that is used for testing and mocking.
+-   The another special case is `LogLlmExecutionToolsWrapper` that is technically also an execution tools but it is more proxy wrapper around other execution tools that logs all calls to execution tools.
 
 #### Script Execution Tools
 
@@ -575,6 +576,12 @@ Execution report is a simple object or markdown that contains information about
 
 <!-- TODO: Write more -->
 
+<!--
+TODO: !!! Write
+### Multiple LLM Tools
+
+-->
+
 ### Remote server
 
 Remote server is a proxy server that uses its execution tools internally and exposes the executor interface externally.

diff --git a/src/_packages/core.index.ts b/src/_packages/core.index.ts
@@ -11,6 +11,7 @@ import { createPromptbookLibraryFromSources } from '../library/constructors/crea
 import { createPromptbookSublibrary } from '../library/constructors/createPromptbookSublibrary';
 import { ExecutionTypes } from '../types/ExecutionTypes';
 import { PROMPTBOOK_VERSION } from '../version';
+import { MultipleLlmExecutionTools } from '../execution/plugins/llm-execution-tools/multiple/MultipleLlmExecutionTools';
 
 // @promptbook/core
 export { ExecutionTypes, PROMPTBOOK_VERSION };
@@ -32,7 +33,7 @@ export { SimplePromptInterfaceTools };
 export { promptbookStringToJson,promptbookJsonToString, validatePromptbookJson };
 
 // @promptbook/executor
-export { createPromptbookExecutor };
+export { createPromptbookExecutor,MultipleLlmExecutionTools };
 
 // @promptbook/callback-prompt
 export { CallbackInterfaceTools, CallbackInterfaceToolsOptions };
diff --git a/src/execution/plugins/llm-execution-tools/anthropic-claude/AnthropicClaudeExecutionTools.ts b/src/execution/plugins/llm-execution-tools/anthropic-claude/AnthropicClaudeExecutionTools.ts
@@ -193,6 +193,7 @@ export class AnthropicClaudeExecutionTools implements LlmExecutionTools {
 }
 
 /**
+ * TODO: [🧠] Maybe handle errors via transformAnthropicError (like transformAzureError)
  * TODO: [🍓][♐] Allow to list compatible models with each variant
  * TODO: Maybe Create some common util for gptChat and gptComplete
  * TODO: Maybe make custom OpenaiError

diff --git a/src/execution/plugins/llm-execution-tools/azure-openai/AzureOpenAiExecutionTools.ts b/src/execution/plugins/llm-execution-tools/azure-openai/AzureOpenAiExecutionTools.ts
@@ -45,69 +45,73 @@ export class AzureOpenAiExecutionTools implements LlmExecutionTools {
             throw new PromptbookExecutionError('Use gptChat only for CHAT variant');
         }
 
-        const modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
-        const modelSettings = {
-            maxTokens: modelRequirements.maxTokens,
-            //                                      <- TODO: Make some global max cap for maxTokens
-            user: this.options.user,
-        };
-
-        const messages = [
-            {
-                role: 'user',
-                content,
-            },
-        ];
-
-        const start: string_date_iso8601 = getCurrentIsoDate();
-        let complete: string_date_iso8601;
-
-        if (this.options.isVerbose) {
-            console.info(colors.bgWhite('messages'), JSON.stringify(messages, null, 4));
-        }
-        const rawResponse = await this.client.getChatCompletions(modelName, messages, modelSettings);
-
-        if (this.options.isVerbose) {
-            console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
-        }
-
-        if (!rawResponse.choices[0]) {
-            throw new PromptbookExecutionError('No choises from Azure OpenAI');
-        }
-
-        if (rawResponse.choices.length > 1) {
-            // TODO: This should be maybe only warning
-            throw new PromptbookExecutionError('More than one choise from Azure OpenAI');
-        }
-
-        if (!rawResponse.choices[0].message || !rawResponse.choices[0].message.content) {
-            throw new PromptbookExecutionError('Empty response from Azure OpenAI');
-        }
-
-        const resultContent = rawResponse.choices[0].message.content;
-        // eslint-disable-next-line prefer-const
-        complete = getCurrentIsoDate();
-        const usage = {
-            price: 'UNKNOWN' /* <- TODO: [🐞] Compute usage */,
-            inputTokens: rawResponse.usage?.promptTokens || 'UNKNOWN',
-            outputTokens: rawResponse.usage?.completionTokens || 'UNKNOWN',
-        } as const;
-
-        if (!resultContent) {
-            throw new PromptbookExecutionError('No response message from OpenAI');
+        try {
+            const modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
+            const modelSettings = {
+                maxTokens: modelRequirements.maxTokens,
+                //                                      <- TODO: Make some global max cap for maxTokens
+                user: this.options.user,
+            };
+
+            const messages = [
+                {
+                    role: 'user',
+                    content,
+                },
+            ];
+
+            const start: string_date_iso8601 = getCurrentIsoDate();
+            let complete: string_date_iso8601;
+
+            if (this.options.isVerbose) {
+                console.info(colors.bgWhite('messages'), JSON.stringify(messages, null, 4));
+            }
+            const rawResponse = await this.client.getChatCompletions(modelName, messages, modelSettings);
+
+            if (this.options.isVerbose) {
+                console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
+            }
+
+            if (!rawResponse.choices[0]) {
+                throw new PromptbookExecutionError('No choises from Azure OpenAI');
+            }
+
+            if (rawResponse.choices.length > 1) {
+                // TODO: This should be maybe only warning
+                throw new PromptbookExecutionError('More than one choise from Azure OpenAI');
+            }
+
+            if (!rawResponse.choices[0].message || !rawResponse.choices[0].message.content) {
+                throw new PromptbookExecutionError('Empty response from Azure OpenAI');
+            }
+
+            const resultContent = rawResponse.choices[0].message.content;
+            // eslint-disable-next-line prefer-const
+            complete = getCurrentIsoDate();
+            const usage = {
+                price: 'UNKNOWN' /* <- TODO: [🐞] Compute usage */,
+                inputTokens: rawResponse.usage?.promptTokens || 'UNKNOWN',
+                outputTokens: rawResponse.usage?.completionTokens || 'UNKNOWN',
+            } as const;
+
+            if (!resultContent) {
+                throw new PromptbookExecutionError('No response message from OpenAI');
+            }
+
+            return {
+                content: resultContent,
+                modelName,
+                timing: {
+                    start,
+                    complete,
+                },
+                usage,
+                rawResponse,
+                // <- [🤹‍♂️]
+            };
+        } catch (error) {
+            throw this.transformAzureError(error);
         }
-
-        return {
-            content: resultContent,
-            modelName,
-            timing: {
-                start,
-                complete,
-            },
-            usage,
-            rawResponse,
-            // <- [🤹‍♂️]
-        };
     }
 
     /**
@@ -125,57 +129,73 @@ export class AzureOpenAiExecutionTools implements LlmExecutionTools {
             throw new PromptbookExecutionError('Use gptComplete only for COMPLETION variant');
         }
 
-        const modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
-        const modelSettings = {
-            maxTokens: modelRequirements.maxTokens || 2000, // <- Note: 2000 is for lagacy reasons
-            //                                                  <- TODO: Make some global max cap for maxTokens
-            user: this.options.user,
-        };
-
-        const start: string_date_iso8601 = getCurrentIsoDate();
-        let complete: string_date_iso8601;
-
-        if (this.options.isVerbose) {
-            console.info(colors.bgWhite('content'), JSON.stringify(content, null, 4));
-        }
-        const rawResponse = await this.client.getCompletions(modelName, [content], modelSettings);
-        if (this.options.isVerbose) {
-            console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
-        }
-
-        if (!rawResponse.choices[0]) {
-            throw new PromptbookExecutionError('No choises from OpenAI');
-        }
-
-        if (rawResponse.choices.length > 1) {
-            // TODO: This should be maybe only warning
-            throw new PromptbookExecutionError('More than one choise from OpenAI');
+        try {
+            const modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
+            const modelSettings = {
+                maxTokens: modelRequirements.maxTokens || 2000, // <- Note: 2000 is for lagacy reasons
+                //                                                  <- TODO: Make some global max cap for maxTokens
+                user: this.options.user,
+            };
+
+            const start: string_date_iso8601 = getCurrentIsoDate();
+            let complete: string_date_iso8601;
+
+            if (this.options.isVerbose) {
+                console.info(colors.bgWhite('content'), JSON.stringify(content, null, 4));
+            }
+            const rawResponse = await this.client.getCompletions(modelName, [content], modelSettings);
+            if (this.options.isVerbose) {
+                console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
+            }
+
+            if (!rawResponse.choices[0]) {
+                throw new PromptbookExecutionError('No choises from OpenAI');
+            }
+
+            if (rawResponse.choices.length > 1) {
+                // TODO: This should be maybe only warning
+                throw new PromptbookExecutionError('More than one choise from OpenAI');
+            }
+
+            const resultContent = rawResponse.choices[0].text;
+            // eslint-disable-next-line prefer-const
+            complete = getCurrentIsoDate();
+            const usage = {
+                price: 'UNKNOWN' /* <- TODO: [🐞] Compute usage */,
+                inputTokens: rawResponse.usage.promptTokens,
+                outputTokens: rawResponse.usage.completionTokens,
+            } as const;
+
+            if (!resultContent) {
+                throw new PromptbookExecutionError('No response message from OpenAI');
+            }
+
+            return {
+                content: resultContent,
+                modelName,
+                timing: {
+                    start,
+                    complete,
+                },
+                usage,
+                rawResponse,
+                // <- [🤹‍♂️]
+            };
+        } catch (error) {
+            throw this.transformAzureError(error);
         }
+    }
 
-        const resultContent = rawResponse.choices[0].text;
-        // eslint-disable-next-line prefer-const
-        complete = getCurrentIsoDate();
-        const usage = {
-            price: 'UNKNOWN' /* <- TODO: [🐞] Compute usage */,
-            inputTokens: rawResponse.usage.promptTokens,
-            outputTokens: rawResponse.usage.completionTokens,
-        } as const;
-
-        if (!resultContent) {
-            throw new PromptbookExecutionError('No response message from OpenAI');
+    /**
+     * Changes Azure error (which is not propper Error but object) to propper Error
+     */
+    private transformAzureError(azureError: { code: string; message: string }): Error {
+        if (typeof azureError !== 'object' || azureError === null) {
+            return new PromptbookExecutionError(`Unknown Azure OpenAI error`);
         }
 
-        return {
-            content: resultContent,
-            modelName,
-            timing: {
-                start,
-                complete,
-            },
-            usage,
-            rawResponse,
-            // <- [🤹‍♂️]
-        };
+        const { code, message } = azureError;
+        return new PromptbookExecutionError(`${code}: ${message}`);
     }
 
     /**

diff --git a/src/execution/plugins/llm-execution-tools/langtail/playground/playground.ts b/src/execution/plugins/llm-execution-tools/langtail/playground/playground.ts
@@ -35,7 +35,7 @@ async function playground() {
     /**/
 
     /*/
-     // TODO: !!!!! This should work
+    // TODO: !!!!! This should work
     const prompt = {
         content: `Hello, my name is Alice.`,
         modelRequirements: {
@@ -48,7 +48,7 @@ async function playground() {
     /**/
 
     /*/
-     // TODO: !!!!! This should work
+    // TODO: !!!!! This should work
     const prompt = {
         content: `Hello, my name is Alice.`,
         modelRequirements: {