diff --git a/apps/server/migrations/20240126075553_add_observer_to_graph_events.ts b/apps/server/migrations/20240126075553_add_observer_to_graph_events.ts index c7ef8da158..65239b9590 100644 --- a/apps/server/migrations/20240126075553_add_observer_to_graph_events.ts +++ b/apps/server/migrations/20240126075553_add_observer_to_graph_events.ts @@ -2,7 +2,7 @@ import { Knex } from 'knex' export async function up(knex: Knex): Promise { await knex.schema.alterTable('graphEvents', (table) => { - table.string('observer').notNullable() + table.string('observer').nullable() }) } diff --git a/packages/client/editor/src/components/PropertiesWindow/CompletionProviderOptions.tsx b/packages/client/editor/src/components/PropertiesWindow/CompletionProviderOptions.tsx new file mode 100644 index 0000000000..179dc9c1c3 --- /dev/null +++ b/packages/client/editor/src/components/PropertiesWindow/CompletionProviderOptions.tsx @@ -0,0 +1,162 @@ +import React, { useState, useEffect } from "react"; +import { ConfigurationComponentProps } from "./PropertiesWindow"; + +import { + allOpenAICompletionModelsArray, + googleAIStudioModelsArray, + togetherAIModelsArray, + // ai21ModelsArray, + // alephAlphaModelsArray, + // anthropicModelsArray, + // anyscaleModelsArray, + // baseTenModelsArray, + // bedrockModelsArray, + // cloudflareWorkersAIModelsArray, + // deepInfraChatModelsArray, + // huggingFaceModelsWithPromptFormattingArray, + // mistralAIModelsArray, + // nlpCloudModelsArray, + // ollamaModelsArray, + // openRouterModelsArray, + // palmModelsArray, + // perplexityAIModelsArray, + // petalsModelsArray, + // replicateModelsArray, + // sageMakerModelsArray, + // vertexAIGoogleModelsArray, + // vllmModelsArray, + // voyageAIModelsArray, + // xinferenceModelsArray +} from "plugins/core/src/lib/services/coreLLMService/constants/completionModelArrays"; +import { CompletionModels } from "plugins/core/src/lib/services/coreLLMService/types/completionModels"; +import { LLMProviders } from 'plugins/core/src/lib/services/coreLLMService/types/providerTypes'; +import { ActiveProviders, activeProviders } from "plugins/core/src/lib/services/coreLLMService/constants/providers"; + +export const CompletionProviderOptions = (props: ConfigurationComponentProps) => { + const [selectedProvider, setSelectedProvider] = useState(props.fullConfig.modelProvider); + const [selectedModel, setSelectedModel] = useState(props.fullConfig.model); + const [filteredModels, setFilteredModels] = useState([]); + + useEffect(() => { + // get all completion models for the selected provider` + switch (selectedProvider) { + case LLMProviders.OpenAI: + setFilteredModels(allOpenAICompletionModelsArray); + break; + case LLMProviders.GoogleAIStudio: + setFilteredModels(googleAIStudioModelsArray); + break; + case LLMProviders.TogetherAI: + setFilteredModels(togetherAIModelsArray); + break; + // case LLMProviders.VertexAI: + // setFilteredModels(vertexAIGoogleModelsArray); + // break; + // case LLMProviders.AI21: + // setFilteredModels(ai21ModelsArray); + // break; + // case LLMProviders.Anthropic: + // setFilteredModels(anthropicModelsArray); + // break; + // case LLMProviders.AlephAlpha: + // setFilteredModels(alephAlphaModelsArray); + // break; + // case LLMProviders.Anyscale: + // setFilteredModels(anyscaleModelsArray); + // break; + // case LLMProviders.Baseten: + // setFilteredModels(baseTenModelsArray); + // break; + // case LLMProviders.Bedrock: + // setFilteredModels(bedrockModelsArray); + // break; + // case LLMProviders.CloudflareWorkersAI: + // setFilteredModels(cloudflareWorkersAIModelsArray); + // break; + // case LLMProviders.DeepInfra: + // setFilteredModels(deepInfraChatModelsArray); + // break; + // case LLMProviders.HuggingFace: + // setFilteredModels(huggingFaceModelsWithPromptFormattingArray); + // break; + // case LLMProviders.Mistral: + // setFilteredModels(mistralAIModelsArray); + // break; + // case LLMProviders.NLPCloud: + // setFilteredModels(nlpCloudModelsArray); + // break; + // case LLMProviders.Ollama: + // setFilteredModels(ollamaModelsArray); + // break; + // case LLMProviders.OpenRouter: + // setFilteredModels(openRouterModelsArray); + // break; + // case LLMProviders.Palm: + // setFilteredModels(palmModelsArray); + // break; + // case LLMProviders.PerplexityAI: + // setFilteredModels(perplexityAIModelsArray); + // break; + // case LLMProviders.Petals: + // setFilteredModels(petalsModelsArray); + // break; + // case LLMProviders.Replicate: + // setFilteredModels(replicateModelsArray); + // break; + // case LLMProviders.Sagemaker: + // setFilteredModels(sageMakerModelsArray); + // break; + // case LLMProviders.VLLM: + // setFilteredModels(vllmModelsArray); + // break; + // case LLMProviders.VoyageAI: + // setFilteredModels(voyageAIModelsArray); + // break; + // case LLMProviders.Xinference: + // setFilteredModels(xinferenceModelsArray); + // break; + default: setFilteredModels([]); + } + }, [selectedProvider]); + + const onProviderChange = (provider: ActiveProviders) => { + setSelectedProvider(provider); + props.updateConfigKey("modelProvider", provider); + }; + + const onModelChange = (model: CompletionModels) => { + setSelectedModel(model); + props.updateConfigKey("model", model); + }; + + return ( +
+

Model Provider

+
+ +
+
+

Model

+
+ +
+
+ ); +} diff --git a/packages/client/editor/src/components/PropertiesWindow/PropertiesWindow.tsx b/packages/client/editor/src/components/PropertiesWindow/PropertiesWindow.tsx index af5517c57a..5f3fb15add 100644 --- a/packages/client/editor/src/components/PropertiesWindow/PropertiesWindow.tsx +++ b/packages/client/editor/src/components/PropertiesWindow/PropertiesWindow.tsx @@ -14,6 +14,8 @@ import { SpellInterface } from 'server/schemas'; import { VariableNames } from './variableNames'; import { ValueType } from './ValueType'; import { DefaultConfig } from './DefaultConfig'; +import { CompletionProviderOptions } from './CompletionProviderOptions'; + type Props = { tab: Tab @@ -38,7 +40,8 @@ const ConfigurationComponents = { eventStateProperties: EventStateProperties, variableNames: VariableNames, valueType: ValueType, - default: DefaultConfig + default: DefaultConfig, + modelProviders: CompletionProviderOptions, } export const PropertiesWindow = (props: Props) => { @@ -47,6 +50,7 @@ export const PropertiesWindow = (props: Props) => { const selectedNode = useSelector(selectActiveNode(props.tab.id)) const handleChange = useChangeNodeData(selectedNode?.id); + if (!selectedNode) return null const spec = nodeSpecs.find(spec => spec.type === selectedNode.type) @@ -109,4 +113,4 @@ export const PropertiesWindow = (props: Props) => { }) as any} ) -} \ No newline at end of file +} diff --git a/packages/shared/nodeSpec/src/nodeSpec.json b/packages/shared/nodeSpec/src/nodeSpec.json index c36b0e0346..5451b4d7b7 100644 --- a/packages/shared/nodeSpec/src/nodeSpec.json +++ b/packages/shared/nodeSpec/src/nodeSpec.json @@ -135,51 +135,9 @@ "defaultValue": "" }, { - "name": "model", + "name": "modelOverride", "valueType": "string", - "defaultValue": "gpt-3.5-turbo", - "choices": [ - { - "text": "gpt-4-1106-preview", - "value": "gpt-4-1106-preview" - }, - { - "text": "gpt-3.5-turbo", - "value": "gpt-3.5-turbo" - }, - { - "text": "gpt-3.5-turbo-1106", - "value": "gpt-3.5-turbo-1106" - }, - { - "text": "gpt-3.5-turbo-0301", - "value": "gpt-3.5-turbo-0301" - }, - { - "text": "gpt-3.5-turbo-0613", - "value": "gpt-3.5-turbo-0613" - }, - { - "text": "gpt-3.5-turbo-16k", - "value": "gpt-3.5-turbo-16k" - }, - { - "text": "gpt-3.5-turbo-16k-0613", - "value": "gpt-3.5-turbo-16k-0613" - }, - { - "text": "gpt-4", - "value": "gpt-4" - }, - { - "text": "gpt-4-0314", - "value": "gpt-4-0314" - }, - { - "text": "gpt-4-0613", - "value": "gpt-4-0613" - } - ] + "defaultValue": "" }, { "name": "stop", @@ -227,15 +185,25 @@ { "name": "stream", "valueType": "string" - }, - { - "name": "modelUsed", - "valueType": "string" } ], "configuration": [ { - "name": "modelChoices", + "name": "modelProviders", + "valueType": "array", + "defaultValue": [ + "openai", + "together", + "google" + ] + }, + { + "name": "modelProvider", + "valueType": "string", + "defaultValue": "openai" + }, + { + "name": "models", "valueType": "array", "defaultValue": [ "gpt-4-1106-preview", @@ -247,7 +215,163 @@ "gpt-3.5-turbo-16k-0613", "gpt-4", "gpt-4-0314", - "gpt-4-0613" + "gpt-4-0613", + "gpt-3.5-turbo-instruct", + "text-davinci-003", + "ada-001", + "curie-001", + "babbage-001", + "babbage-002", + "davinci-002", + "gpt-4-vision-preview", + "mistralai/Mistral-7B-Instruct-v0.1", + "meta-llama/Llama-2-7b-chat", + "tiiuae/falcon-7b-instruct", + "mosaicml/mpt-7b-chat", + "codellama/CodeLlama-34b-Instruct-hf", + "WizardLM/WizardCoder-Python-34B-V1.0", + "Phind/Phind-CodeLlama-34B-v2", + "ollama/mistral", + "ollama/llama2", + "ollama/llama2:13b", + "ollama/llama2:70b", + "ollama/llama2-uncensored", + "ollama/codellama", + "ollama/orca-mini", + "ollama/vicuna", + "ollama/nous-hermes", + "ollama/nous-hermes:13b", + "ollama/wizard-vicuna", + "ollama/llama-va", + "gemini-pro", + "gemini-pro-vision", + "chat-bison", + "gemini-pro", + "gemini-pro-vision", + "mistral-tiny", + "mistral-small", + "mistral-medium", + "claude-2.1", + "claude-2", + "claude-instant-1", + "claude-instant-1.2", + "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-7b", + "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-7b-f", + "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-13b", + "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-13b-f", + "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-70b", + "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-70b-b-f", + "anthropic.claude-v2:1", + "anthropic.claude-v2", + "anthropic.claude-instant-v1", + "anthropic.claude-v1", + "amazon.titan-text-lite-v1", + "amazon.titan-text-express-v1", + "cohere.command-text-v14", + "ai21.j2-mid-v1", + "ai21.j2-ultra-v1", + "meta.llama2-13b-chat-v1", + "meta.llama2-70b-chat-v1", + "pplx-7b-chat", + "pplx-70b-chat", + "pplx-7b-online", + "pplx-70b-online", + "codellama-34b-instruct", + "llama-2-13b-chat", + "llama-2-70b-chat", + "mistral-7b-instruct", + "openhermes-2-mistral-7b", + "openhermes-2.5-mistral-7b", + "pplx-7b-chat-alpha", + "pplx-70b-chat-alpha", + "vllm/meta-llama/Llama-2-7b", + "vllm/tiiuae/falcon-7b-instruct", + "vllm/mosaicml/mpt-7b-chat", + "vllm/codellama/CodeLlama-34b-Instruct-hf", + "vllm/WizardLM/WizardCoder-Python-34B-V1.0", + "vllm/Phind/Phind-CodeLlama-34B-v2", + "xinference/bge-base-en", + "xinference/bge-base-en-v1.5", + "xinference/bge-base-zh", + "xinference/bge-base-zh-v1.5", + "xinference/bge-large-en", + "xinference/bge-large-en-v1.5", + "xinference/bge-large-zh", + "xinference/bge-large-zh-noinstruct", + "xinference/bge-large-zh-v1.5", + "xinference/bge-small-en-v1.5", + "xinference/bge-small-zh", + "xinference/bge-small-zh-v1.5", + "xinference/e5-large-v2", + "xinference/gte-base", + "xinference/gte-large", + "xinference/jina-embeddings-v2-base-en", + "xinference/jina-embeddings-v2-small-en", + "xinference/multilingual-e5-large", + "@cf/meta/llama-2-7b-chat-fp16", + "@cf/meta/llama-2-7b-chat-int8", + "@cf/mistral/mistral-7b-instruct-v0.1", + "@hf/thebloke/codellama-7b-instruct-awq", + "j2-light", + "j2-mid", + "j2-ultra", + "dolphin", + "chatDolphin", + "deepinfra/meta-llama/Llama-2-70b-chat-hf", + "deepinfra/meta-llama/Llama-2-7b-chat-hf", + "deepinfra/meta-llama/Llama-2-13b-chat-hf", + "deepinfra/codellama/CodeLlama-34b-Instruct-hf", + "deepinfra/mistralai/Mistral-7B-Instruct-v0.1", + "deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1", + "togethercomputer/llama-2-70b-chat", + "togethercomputer/llama-2-70b", + "togethercomputer/LLaMA-2-7B-32K", + "togethercomputer/Llama-2-7B-32K-Instruct", + "togethercomputer/llama-2-7b", + "togethercomputer/falcon-40b-instruct", + "togethercomputer/falcon-7b-instruct", + "togethercomputer/alpaca-7b", + "HuggingFaceH4/starchat-alpha", + "togethercomputer/CodeLlama-34b", + "togethercomputer/CodeLlama-34b-Instruct", + "togethercomputer/CodeLlama-34b-Python", + "defog/sqlcoder", + "NumbersStation/nsql-llama-2-7B", + "WizardLM/WizardCoder-15B-V1.0", + "WizardLM/WizardCoder-Python-34B-V1.0", + "NousResearch/Nous-Hermes-Llama2-13b", + "Austism/chronos-hermes-13b", + "upstage/SOLAR-0-70b-16bit", + "WizardLM/WizardLM-70B-V1.0", + "baseten/qvv0xeq", + "baseten/q841o8w", + "baseten/31dxrj3", + "petals/petals-team/StableBeluga2", + "petals/huggyllama/llama-65b", + "voyage-01", + "voyage-lite-01", + "voyage-lite-01-instruct", + "luminous-base", + "luminous-base-control", + "luminous-extended", + "luminous-extended-control", + "luminous-supreme", + "luminous-supreme-control" + ] + }, + { + "name": "model", + "valueType": "string", + "defaultValue": "gpt-3.5-turbo" + }, + { + "name": "hiddenProperties", + "valueType": "string", + "defaultValue": [ + "hiddenProperties", + "modelProvider", + "model", + "models" ] } ] diff --git a/plugins/core/src/lib/corePlugin.ts b/plugins/core/src/lib/corePlugin.ts index df871947b1..0d7f12a1b9 100644 --- a/plugins/core/src/lib/corePlugin.ts +++ b/plugins/core/src/lib/corePlugin.ts @@ -16,7 +16,7 @@ import { sendMessage } from './nodes/actions/sendMessage' import { textTemplate } from './nodes/functions/textTemplate' import { registerStructProfile } from './registerStructProfile' import { streamMessage } from './nodes/actions/streamMessage' -import { LLMProviderKeys } from './services/coreLLMService/types' +import { LLMProviderKeys } from './services/coreLLMService/types/providerTypes' import { variableGet } from './nodes/query/variableGet' import { VariableService } from './services/variableService' import { variableSet } from './nodes/query/variableSet' @@ -139,6 +139,13 @@ export class CorePlugin extends CoreEventsPlugin< }) } + async initializeFunctionalities() { + await this.getLLMCredentials() + + this.centralEventBus.on(ON_MESSAGE, this.handleOnMessage.bind(this)) + this.client.onMessage(this.handleOnMessage.bind(this)) + } + /** * Defines the dependencies that the plugin will use. Creates a new set of dependencies every time. */ @@ -219,13 +226,6 @@ export class CorePlugin extends CoreEventsPlugin< return registerStructProfile(coreRegistry, logger) } - async initializeFunctionalities() { - await this.getLLMCredentials() - - this.centralEventBus.on(ON_MESSAGE, this.handleOnMessage.bind(this)) - this.client.onMessage(this.handleOnMessage.bind(this)) - } - handleOnMessage(payload: EventPayload) { const event = this.formatMessageEvent(ON_MESSAGE, payload) this.emitEvent(ON_MESSAGE, event) diff --git a/plugins/core/src/lib/nodes/actions/generateText.ts b/plugins/core/src/lib/nodes/actions/generateText.ts index 5cd1df1670..35bb7e6764 100644 --- a/plugins/core/src/lib/nodes/actions/generateText.ts +++ b/plugins/core/src/lib/nodes/actions/generateText.ts @@ -4,21 +4,44 @@ import { makeFlowNodeDefinition, } from '@magickml/behave-graph' import { CoreLLMService } from '../../services/coreLLMService/coreLLMService' -import { - LLMModels, - OpenAIChatCompletionModels, -} from '../../services/coreLLMService/types' +import { allCompletionModels } from '../../services/coreLLMService/constants/allCompletionModels' + import { CORE_DEP_KEYS } from '../../constants' import { IEventStore } from 'server/grimoire' +import { + OpenAIChatCompletionModels, + CompletionModels, +} from '../../services/coreLLMService/types/completionModels' + +import { activeProviders } from '../../services/coreLLMService/constants/providers' +import { LLMProviders } from '../../services/coreLLMService/types/providerTypes' export const generateText = makeFlowNodeDefinition({ typeName: 'magick/generateText', category: NodeCategory.Action, label: 'Generate Text', configuration: { - modelChoices: { + modelProviders: { + valueType: 'array', + defaultValue: activeProviders, + }, + modelProvider: { + valueType: 'string', + //TODO: default to google when ready + defaultValue: LLMProviders.OpenAI, + }, + models: { valueType: 'array', - defaultValue: Object.values(OpenAIChatCompletionModels), + defaultValue: allCompletionModels, + }, + model: { + valueType: 'string', + //TODO: default to google when ready + defaultValue: OpenAIChatCompletionModels.GPT35Turbo, + }, + hiddenProperties: { + valueType: 'string', + defaultValue: ['hiddenProperties', 'modelProvider', 'model', 'models'], }, }, in: { @@ -27,10 +50,9 @@ export const generateText = makeFlowNodeDefinition({ valueType: 'string', defaultValue: '', }, - model: { + modelOverride: { valueType: 'string', - choices: Object.values(OpenAIChatCompletionModels) as string[], - defaultValue: OpenAIChatCompletionModels.GPT35Turbo, + defaultValue: '', }, stop: { valueType: 'string', @@ -59,10 +81,15 @@ export const generateText = makeFlowNodeDefinition({ done: 'flow', onStream: 'flow', stream: 'string', - modelUsed: 'string', }, initialState: undefined, - triggered: async ({ commit, read, write, graph: { getDependency } }) => { + triggered: async ({ + commit, + read, + write, + configuration, + graph: { getDependency }, + }) => { /** * This promise wrapper is super important. If we await the whole LLM * completion, it will block continued fiber execution, which will prevent @@ -83,8 +110,8 @@ export const generateText = makeFlowNodeDefinition({ throw new Error('No coreLLMService provided') } - const model: LLMModels = - read('model') || OpenAIChatCompletionModels.GPT35Turbo + const model: CompletionModels = + read('modelOverride') || configuration.model const prompt: string = read('prompt') || '' const temperature: number = read('temperature') || 0.5 const top_p: number = read('top_p') || 1 @@ -106,7 +133,6 @@ export const generateText = makeFlowNodeDefinition({ if (result.done) { write('response', result.value.choices[0].message.content || '') write('completionResponse', result.value) // Assuming the last value is the completion response - write('modelUsed', model) commit('done') } else { const chunk = result.value diff --git a/plugins/core/src/lib/services/coreBudgetManagerService/coreBudgetMangerService.ts b/plugins/core/src/lib/services/coreBudgetManagerService/coreBudgetMangerService.ts index dbd961d17c..3e00737838 100644 --- a/plugins/core/src/lib/services/coreBudgetManagerService/coreBudgetMangerService.ts +++ b/plugins/core/src/lib/services/coreBudgetManagerService/coreBudgetMangerService.ts @@ -1,6 +1,5 @@ import { python } from 'pythonia' -import { CompletionResponse, LLMModels, Message } from '../coreLLMService/types' import { UserService } from '../userService/userService' import { BudgetDuration, @@ -9,6 +8,10 @@ import { } from '../types' import { CLOUD_AGENT_KEY, PORTAL_URL } from 'shared/config' +import { CompletionModels } from '../coreLLMService/types/completionModels' +import { CompletionResponse } from '../coreLLMService/types/completionTypes' +import { Message } from '../coreLLMService/types/messageTypes' + export class CoreBudgetManagerService implements ICoreBudgetManagerService { private liteLLMBudgetManager: IBudgetManagerService | undefined @@ -151,7 +154,9 @@ export class CoreBudgetManagerService implements ICoreBudgetManagerService { * @param projectId - Project ID * @returns Promise */ - async getModelCost(projectId: string): Promise> { + async getModelCost( + projectId: string + ): Promise> { const modelCost = await this.liteLLMBudgetManager?.get_model_cost(projectId) if (modelCost === null || modelCost === undefined) { throw new Error('Error getting model cost') diff --git a/plugins/core/src/lib/services/coreEmbeddingService/constants.ts b/plugins/core/src/lib/services/coreEmbeddingService/constants.ts deleted file mode 100644 index 8ee8764cbc..0000000000 --- a/plugins/core/src/lib/services/coreEmbeddingService/constants.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { LLMProviders } from '../coreLLMService/types' -import { - BedrockEmbeddingModels, - CohereEmbeddingModels, - EmbeddingModels, - HuggingFaceEmbeddingModels, - MistralEmbeddingModels, - OpenAIEmbeddingModels, - VoyageEmbeddingModels, -} from './types' - -export const embeddingProviderMap: Record = { - // Bedrock models - [BedrockEmbeddingModels.AmazonTitanEmbedTextV1]: LLMProviders.Bedrock, - [BedrockEmbeddingModels.CohereEmbedEnglishV3]: LLMProviders.Bedrock, - [BedrockEmbeddingModels.CohereEmbedMultilingualV3]: LLMProviders.Bedrock, - - // Cohere models - [CohereEmbeddingModels.EmbedEnglishLightV20]: LLMProviders.Cohere, - [CohereEmbeddingModels.EmbedEnglishLightV30]: LLMProviders.Cohere, - [CohereEmbeddingModels.EmbedEnglishV20]: LLMProviders.Cohere, - [CohereEmbeddingModels.EmbedEnglishV30]: LLMProviders.Cohere, - [CohereEmbeddingModels.EmbedMultilingualLightV30]: LLMProviders.Cohere, - [CohereEmbeddingModels.EmbedMultilingualV20]: LLMProviders.Cohere, - [CohereEmbeddingModels.EmbedMultilingualV30]: LLMProviders.Cohere, - - // Mistral models - [MistralEmbeddingModels.MistralEmbed]: LLMProviders.Mistral, - - // Voyage models - [VoyageEmbeddingModels.Voyage02]: LLMProviders.VoyageAI, - [VoyageEmbeddingModels.VoyageCode02]: LLMProviders.VoyageAI, - [VoyageEmbeddingModels.VoyageLite01Instruct]: LLMProviders.VoyageAI, - - // OpenAI models - [OpenAIEmbeddingModels.TextEmbeddingAda002]: LLMProviders.OpenAI, - - // HuggingFace models - [HuggingFaceEmbeddingModels.HuggingFaceMicrosoftCodebertBase]: - LLMProviders.HuggingFace, - [HuggingFaceEmbeddingModels.HuggingFaceBAAIBgeLargeZh]: - LLMProviders.HuggingFace, - [HuggingFaceEmbeddingModels.HuggingFaceAnyHfEmbeddingModel]: - LLMProviders.HuggingFace, -} - -const bedrockEmbeddingModelsArray = Object.values(BedrockEmbeddingModels) -const cohereEmbeddingModelsArray = Object.values(CohereEmbeddingModels) -const mistralEmbeddingModelsArray = Object.values(MistralEmbeddingModels) -const voyageEmbeddingModelsArray = Object.values(VoyageEmbeddingModels) -const openAIEmbeddingModelsArray = Object.values(OpenAIEmbeddingModels) -const huggingFaceEmbeddingModelsArray = Object.values( - HuggingFaceEmbeddingModels -) - -export const allEmbeddingModels: EmbeddingModels[] = [ - ...bedrockEmbeddingModelsArray, - ...cohereEmbeddingModelsArray, - ...mistralEmbeddingModelsArray, - ...voyageEmbeddingModelsArray, - ...openAIEmbeddingModelsArray, - ...huggingFaceEmbeddingModelsArray, -] diff --git a/plugins/core/src/lib/services/coreEmbeddingService/coreEmbeddingService.ts b/plugins/core/src/lib/services/coreEmbeddingService/coreEmbeddingService.ts index 4fd7b70858..f17a3671a0 100644 --- a/plugins/core/src/lib/services/coreEmbeddingService/coreEmbeddingService.ts +++ b/plugins/core/src/lib/services/coreEmbeddingService/coreEmbeddingService.ts @@ -12,7 +12,7 @@ import { OpenAIEmbeddingOptions, VoyageEmbeddingOptions, } from './types' -import { LLMProviders } from '../coreLLMService/types' +import { LLMProviders } from '../coreLLMService/types/providerTypes' export class EmbeddingService implements IEmbeddingService { protected liteLLM: any diff --git a/plugins/core/src/lib/services/coreEmbeddingService/types.ts b/plugins/core/src/lib/services/coreEmbeddingService/types.ts index c09610064b..ee6f7ffde4 100644 --- a/plugins/core/src/lib/services/coreEmbeddingService/types.ts +++ b/plugins/core/src/lib/services/coreEmbeddingService/types.ts @@ -1,9 +1,12 @@ import { - AzureModel, - BedrockModels, - CustomOpenAIModel, - LLMProviders, -} from '../coreLLMService/types' + AzureSlug, + BedrockSlug, + CustomOpenAISlug, + HuggingFaceSlug, + MistralAISlug, + VoyageAISlug, +} from '../coreLLMService/types/completionModels' +import { LLMProviders } from '../coreLLMService/types/providerTypes' export interface BaseEmbeddingOptions { model: EmbeddingModels | string @@ -55,7 +58,7 @@ export type GenerateEmbeddingParams = // Azure-specific embedding options export interface AzureEmbeddingOptions extends BaseEmbeddingOptions { - model: AzureModel + model: AzureSlug api_key: string api_base: string api_version: string @@ -67,13 +70,13 @@ export interface OpenAIEmbeddingOptions extends BaseEmbeddingOptions { // Use this for calling /embedding endpoints on OpenAI Compatible Servers export interface CustomOpenAIEmbeddingOptions extends BaseEmbeddingOptions { - model: CustomOpenAIModel + model: CustomOpenAISlug api_base: string // The API endpoint "http://0.0.0.0:8000/" } // Bedrock Embedding Options export interface BedrockEmbeddingOptions extends BaseEmbeddingOptions { - models: BedrockModels + models: BedrockSlug aws_access_key_id?: string aws_secret_access_key?: string aws_region_name?: string @@ -87,17 +90,18 @@ export interface CohereEmbeddingOptions extends BaseEmbeddingOptions { // HuggingFace-specific embedding options export interface HuggingFaceEmbeddingOptions extends BaseEmbeddingOptions { + model: HuggingFaceSlug api_base: string // The API endpoint "https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud" } // Mistral AI Embedding Options export interface MistralEmbeddingOptions extends BaseEmbeddingOptions { - model: MistralEmbeddingModels + model: MistralAISlug } // Voyage-specific embedding options export interface VoyageEmbeddingOptions extends BaseEmbeddingOptions { - model: VoyageEmbeddingModels + model: VoyageAISlug } // OpenAI embedding options @@ -123,6 +127,14 @@ interface Usage { total_tokens: number } +export type EmbeddingModels = + | OpenAIEmbeddingModels + | BedrockEmbeddingModels + | CohereEmbeddingModels + | HuggingFaceEmbeddingModels + | MistralEmbeddingModels + | VoyageEmbeddingModels + export interface IEmbeddingService { initialize(): Promise generateEmbedding( @@ -168,7 +180,7 @@ export enum CohereEmbeddingModels { } export enum MistralEmbeddingModels { - MistralEmbed = 'mistral/mistral-embed', + MistralEmbed = 'mistral-embed', } export enum VoyageEmbeddingModels { @@ -182,14 +194,7 @@ export enum OpenAIEmbeddingModels { } export enum HuggingFaceEmbeddingModels { - HuggingFaceMicrosoftCodebertBase = 'huggingface/microsoft/codebert-base', - HuggingFaceBAAIBgeLargeZh = 'huggingface/BAAI/bge-large-zh', - HuggingFaceAnyHfEmbeddingModel = 'huggingface/any-hf-embedding-model', + HuggingFaceMicrosoftCodebertBase = 'microsoft/codebert-base', + HuggingFaceBAAIBgeLargeZh = 'BAAI/bge-large-zh', + HuggingFaceAnyHfEmbeddingModel = 'any-hf-embedding-model', } -export type EmbeddingModels = - | BedrockEmbeddingModels - | CohereEmbeddingModels - | MistralEmbeddingModels - | VoyageEmbeddingModels - | OpenAIEmbeddingModels - | HuggingFaceEmbeddingModels diff --git a/plugins/core/src/lib/services/coreLLMService/constants.ts b/plugins/core/src/lib/services/coreLLMService/constants.ts deleted file mode 100644 index e846277c77..0000000000 --- a/plugins/core/src/lib/services/coreLLMService/constants.ts +++ /dev/null @@ -1,316 +0,0 @@ -import { allEmbeddingModels } from '../coreEmbeddingService/constants' -import { EmbeddingModels } from '../coreEmbeddingService/types' -import { - AI21Models, - AlephAlphaModels, - AnthropicModels, - BaseTenModels, - BedrockModels, - CloudflareWorkersAIModels, - DeepInfraChatModels, - GoogleAIStudioModels, - HuggingFaceModelsWithPromptFormatting, - LLMModels, - LLMProviderKeys, - MistralAIModels, - NLPCloudModels, - OllamaModels, - OllamaVisionModels, - OpenAIChatCompletionModels, - OpenAITextCompletionInstructModels, - OpenAIVisionModels, - PalmModels, - PerplexityAIModels, - PetalsModels, - SageMakerModels, - TogetherAIModels, - VLLMModels, - VertexAIGoogleModels, - VoyageAIModels, - XinferenceModels, -} from './types' - -export const modelProviderMap: Record = { - /// OpenAI Chat Completion Models - [OpenAIChatCompletionModels.GPT35Turbo1106Preview]: LLMProviderKeys.OpenAI, - [OpenAIChatCompletionModels.GPT35Turbo]: LLMProviderKeys.OpenAI, - [OpenAIChatCompletionModels.GPT35Turbo1106]: LLMProviderKeys.OpenAI, - [OpenAIChatCompletionModels.GPT35Turbo0301]: LLMProviderKeys.OpenAI, - [OpenAIChatCompletionModels.GPT35Turbo0613]: LLMProviderKeys.OpenAI, - [OpenAIChatCompletionModels.GPT35Turbo16k]: LLMProviderKeys.OpenAI, - [OpenAIChatCompletionModels.GPT35Turbo16k0613]: LLMProviderKeys.OpenAI, - [OpenAIChatCompletionModels.GPT4]: LLMProviderKeys.OpenAI, - [OpenAIChatCompletionModels.GPT40314]: LLMProviderKeys.OpenAI, - [OpenAIChatCompletionModels.GPT40613]: LLMProviderKeys.OpenAI, - - // OpenAI Text Completion / Instruct Models - [OpenAITextCompletionInstructModels.Ada001]: LLMProviderKeys.OpenAI, - [OpenAITextCompletionInstructModels.Babbage001]: LLMProviderKeys.OpenAI, - [OpenAITextCompletionInstructModels.Babbage002]: LLMProviderKeys.OpenAI, - [OpenAITextCompletionInstructModels.Curie001]: LLMProviderKeys.OpenAI, - [OpenAITextCompletionInstructModels.Davinci002]: LLMProviderKeys.OpenAI, - [OpenAITextCompletionInstructModels.GPT35TurboInstruct]: - LLMProviderKeys.OpenAI, - [OpenAITextCompletionInstructModels.TextDavinci003]: LLMProviderKeys.OpenAI, - - // OpenAI Vision Models - [OpenAIVisionModels.GPT4VisionPreview]: LLMProviderKeys.OpenAI, - - // HuggingFace Models With Prompt Formatting - [HuggingFaceModelsWithPromptFormatting.HuggingFaceCodellamaCodeLlama34bInstructHf]: - LLMProviderKeys.HuggingFace, - [HuggingFaceModelsWithPromptFormatting.HuggingFaceMetaLlamaLlama27bChat]: - LLMProviderKeys.HuggingFace, - [HuggingFaceModelsWithPromptFormatting.HuggingFaceMistral7BInstructV01]: - LLMProviderKeys.HuggingFace, - [HuggingFaceModelsWithPromptFormatting.HuggingFaceMosaicmlMpt7bChat]: - LLMProviderKeys.HuggingFace, - [HuggingFaceModelsWithPromptFormatting.HuggingFacePhindPhindCodeLlama34Bv2]: - LLMProviderKeys.HuggingFace, - [HuggingFaceModelsWithPromptFormatting.HuggingFaceTiiuaeFalcon7bInstruct]: - LLMProviderKeys.HuggingFace, - [HuggingFaceModelsWithPromptFormatting.HuggingFaceWizardLMWizardCoderPython34BV10]: - LLMProviderKeys.HuggingFace, - - // Ollama Models - [OllamaModels.OlamaMistral]: LLMProviderKeys.Ollama, - [OllamaModels.OlamaLlama27B]: LLMProviderKeys.Ollama, - [OllamaModels.OlamaLlama213B]: LLMProviderKeys.Ollama, - [OllamaModels.OlamaLlama270B]: LLMProviderKeys.Ollama, - [OllamaModels.OlamaLlama2Uncensored]: LLMProviderKeys.Ollama, - [OllamaModels.OlamaCodeLlama]: LLMProviderKeys.Ollama, - [OllamaModels.OlamaNousHermes]: LLMProviderKeys.Ollama, - [OllamaModels.OlamaOrcaMini]: LLMProviderKeys.Ollama, - [OllamaModels.OlamaVicuna]: LLMProviderKeys.Ollama, - [OllamaModels.OlamaNousHermes13B]: LLMProviderKeys.Ollama, - [OllamaModels.OlamaWizardVicunaUncensored]: LLMProviderKeys.Ollama, - - // Ollama Vision Models - [OllamaVisionModels.LLAVA]: LLMProviderKeys.Ollama, - - // VertexAI - [VertexAIGoogleModels.GeminiPro]: LLMProviderKeys.VertexAI, - [VertexAIGoogleModels.GeminiProVision]: LLMProviderKeys.VertexAI, - - //Palm - [PalmModels.ChatBison]: LLMProviderKeys.Palm, - - // Google AI Studio - [GoogleAIStudioModels.GeminiGeminiPro]: LLMProviderKeys.GoogleAIStudio, - [GoogleAIStudioModels.GeminiGeminiProVision]: LLMProviderKeys.GoogleAIStudio, - - // Mistral AI Models - [MistralAIModels.MistralTiny]: LLMProviderKeys.Mistral, - [MistralAIModels.MistralSmall]: LLMProviderKeys.Mistral, - [MistralAIModels.MistralMedium]: LLMProviderKeys.Mistral, - - // Anthropic Models - [AnthropicModels.Claude21]: LLMProviderKeys.Anthropic, - [AnthropicModels.Claude2]: LLMProviderKeys.Anthropic, - [AnthropicModels.ClaudeInstant1]: LLMProviderKeys.Anthropic, - [AnthropicModels.ClaudeInstant12]: LLMProviderKeys.Anthropic, - - // SageMaker Models - [SageMakerModels.MetaLlama213B]: LLMProviderKeys.Sagemaker, - [SageMakerModels.MetaLlama213BChatFineTuned]: LLMProviderKeys.Sagemaker, - [SageMakerModels.MetaLlama270B]: LLMProviderKeys.Sagemaker, - [SageMakerModels.MetaLlama270BChatFineTuned]: LLMProviderKeys.Sagemaker, - [SageMakerModels.MetaLlama27B]: LLMProviderKeys.Sagemaker, - [SageMakerModels.MetaLlama27BChatFineTuned]: LLMProviderKeys.Sagemaker, - - // Bedrock Models - [BedrockModels.BedrockAI21J2Mid]: LLMProviderKeys.Bedrock, - [BedrockModels.BedrockAI21J2Ultra]: LLMProviderKeys.Bedrock, - [BedrockModels.BedrockAmazonTitanExpress]: LLMProviderKeys.Bedrock, - [BedrockModels.BedrockAnthropicClaudeInstantV1]: LLMProviderKeys.Bedrock, - [BedrockModels.BedrockAnthropicClaudeV1]: LLMProviderKeys.Bedrock, - [BedrockModels.BedrockAnthropicClaudeV2]: LLMProviderKeys.Bedrock, - [BedrockModels.BedrockAnthropicClaudeV21]: LLMProviderKeys.Bedrock, - [BedrockModels.BedrockCohereCommand]: LLMProviderKeys.Bedrock, - [BedrockModels.BedrockMetaLlama2Chat13b]: LLMProviderKeys.Bedrock, - [BedrockModels.BedrockMetaLlama2Chat70b]: LLMProviderKeys.Bedrock, - [BedrockModels.BedrockAmazonTitanLite]: LLMProviderKeys.Bedrock, - - // Perplexity AI Models - [PerplexityAIModels.Pplx70bChat]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.Pplx70bChatAlpha]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.Pplx70bOnline]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.Pplx7bOnline]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.PplxCodeLlama34bInstruct]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.PplxLlama213bChat]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.PplxLlama270bChat]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.PplxMistral7bInstruct]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.PplxOpenhermes25Mistral7b]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.PplxOpenhermes2Mistral7b]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.Pplx7bChat]: LLMProviderKeys.PerplexityAI, - [PerplexityAIModels.Pplx7bChatAlpha]: LLMProviderKeys.PerplexityAI, - - // VLLM Models - [VLLMModels.CodellamaCodeLlama34bInstructHf]: LLMProviderKeys.VLLM, - [VLLMModels.MetaLlamaLlama27bChat]: LLMProviderKeys.VLLM, - [VLLMModels.MosaicmlMpt7bChat]: LLMProviderKeys.VLLM, - [VLLMModels.PhindPhindCodeLlama34Bv2]: LLMProviderKeys.VLLM, - [VLLMModels.TiiuaeFalcon7bInstruct]: LLMProviderKeys.VLLM, - [VLLMModels.WizardLMWizardCoderPython34BV10]: LLMProviderKeys.VLLM, - - //Xinference Models - [XinferenceModels.BgeBaseEn]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeBaseEnV15]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeBaseZh]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeLargeEn]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeLargeEnV15]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeLargeZh]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeLargeZhNoinstruct]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeLargeZhV15]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeSmallEnV15]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeSmallZh]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeSmallZhV15]: LLMProviderKeys.Xinference, - [XinferenceModels.E5LargeV2]: LLMProviderKeys.Xinference, - [XinferenceModels.GteBase]: LLMProviderKeys.Xinference, - [XinferenceModels.GteLarge]: LLMProviderKeys.Xinference, - [XinferenceModels.JinaEmbeddingsV2BaseEn]: LLMProviderKeys.Xinference, - [XinferenceModels.JinaEmbeddingsV2SmallEn]: LLMProviderKeys.Xinference, - [XinferenceModels.MultilingualE5Large]: LLMProviderKeys.Xinference, - [XinferenceModels.BgeBaseZhV15]: LLMProviderKeys.Xinference, - - // Cloudflare Workers AI Models - [CloudflareWorkersAIModels.MetaLlama27bChatFp16]: - LLMProviderKeys.CloudflareWorkersAI, - [CloudflareWorkersAIModels.MetaLlama27bChatInt8]: - LLMProviderKeys.CloudflareWorkersAI, - [CloudflareWorkersAIModels.Mistral7bInstructV01]: - LLMProviderKeys.CloudflareWorkersAI, - [CloudflareWorkersAIModels.TheBlokeCodellama7bInstructAwq]: - LLMProviderKeys.CloudflareWorkersAI, - - // AI21 Models - [AI21Models.J2Light]: LLMProviderKeys.AI21, - [AI21Models.J2Mid]: LLMProviderKeys.AI21, - [AI21Models.J2Ultra]: LLMProviderKeys.AI21, - - //NLPCloud Models - [NLPCloudModels.ChatDolphin]: LLMProviderKeys.NLPCloud, - [NLPCloudModels.Dolphin]: LLMProviderKeys.NLPCloud, - - // Deep Infra Models - [DeepInfraChatModels.CodellamaCodeLlama34bInstructHf]: - LLMProviderKeys.DeepInfra, - [DeepInfraChatModels.JondurbinAiroborosL270bGpt4141]: - LLMProviderKeys.DeepInfra, - [DeepInfraChatModels.MetaLlamaLlama213bChatHf]: LLMProviderKeys.DeepInfra, - [DeepInfraChatModels.MetaLlamaLlama270bChatHf]: LLMProviderKeys.DeepInfra, - [DeepInfraChatModels.MetaLlamaLlama27bChatHf]: LLMProviderKeys.DeepInfra, - [DeepInfraChatModels.MistralaiMistral7BInstructV01]: - LLMProviderKeys.DeepInfra, - - // Together AI Models - [TogetherAIModels.TogetherLlama270bChat]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherLlama270b]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherLlama27B32K]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherLlama27B32KInstruct]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherLlama27b]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherFalcon40bInstruct]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherFalcon7bInstruct]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherAlpaca7b]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherStarchatAlpha]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherCodeLlama34b]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherCodeLlama34bInstruct]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherCodeLlama34bPython]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherSqlCoder]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherNSQLLlama27B]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherWizardCoder15BV10]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherWizardCoderPython34BV10]: - LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherNousHermesLlama213b]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherChronosHermes13b]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherSolar070b16bit]: LLMProviderKeys.TogetherAI, - [TogetherAIModels.TogetherWizardLM70BV10]: LLMProviderKeys.TogetherAI, - - // Base Ten Models - [BaseTenModels.Falcon7B]: LLMProviderKeys.Baseten, - [BaseTenModels.MPT7BBase]: LLMProviderKeys.Baseten, - [BaseTenModels.WizardLM]: LLMProviderKeys.Baseten, - - // Petals Models - [PetalsModels.HuggyLlamaLlama65]: LLMProviderKeys.Petals, - [PetalsModels.PetalsTeamStableBeluga]: LLMProviderKeys.Petals, - - // Voyage AI Models - [VoyageAIModels.Voyage01]: LLMProviderKeys.VoyageAI, - [VoyageAIModels.VoyageLite01]: LLMProviderKeys.VoyageAI, - [VoyageAIModels.VoyageLite01Instruct]: LLMProviderKeys.VoyageAI, - - // Aleph Alpha Models - [AlephAlphaModels.LuminousBase]: LLMProviderKeys.AlephAlpha, - [AlephAlphaModels.LuminousBaseControl]: LLMProviderKeys.AlephAlpha, - [AlephAlphaModels.LuminousExtended]: LLMProviderKeys.AlephAlpha, - [AlephAlphaModels.LuminousExtendedControl]: LLMProviderKeys.AlephAlpha, - [AlephAlphaModels.LuminousSupreme]: LLMProviderKeys.AlephAlpha, - [AlephAlphaModels.LuminousSupremeControl]: LLMProviderKeys.AlephAlpha, -} - -const openAIChatCompletionModelsArray = Object.values( - OpenAIChatCompletionModels -) -const openAITextCompletionInstructModelsArray = Object.values( - OpenAITextCompletionInstructModels -) -const openAIVisionModelsArray = Object.values(OpenAIVisionModels) -const huggingFaceModelsWithPromptFormattingArray = Object.values( - HuggingFaceModelsWithPromptFormatting -) -const ollamaModelsArray = Object.values(OllamaModels) -const ollamaVisionModelsArray = Object.values(OllamaVisionModels) -const vertexAIGoogleModelsArray = Object.values(VertexAIGoogleModels) -const palmModelsArray = Object.values(PalmModels) -const googleAIStudioModelsArray = Object.values(GoogleAIStudioModels) -const mistralAIModelsArray = Object.values(MistralAIModels) -const anthropicModelsArray = Object.values(AnthropicModels) -const sageMakerModelsArray = Object.values(SageMakerModels) -const bedrockModelsArray = Object.values(BedrockModels) -const perplexityAIModelsArray = Object.values(PerplexityAIModels) -const vllmModelsArray = Object.values(VLLMModels) -const xinferenceModelsArray = Object.values(XinferenceModels) -const cloudflareWorkersAIModelsArray = Object.values(CloudflareWorkersAIModels) -const ai21ModelsArray = Object.values(AI21Models) -const nlpCloudModelsArray = Object.values(NLPCloudModels) -const deepInfraChatModelsArray = Object.values(DeepInfraChatModels) -const togetherAIModelsArray = Object.values(TogetherAIModels) -const baseTenModelsArray = Object.values(BaseTenModels) -const petalsModelsArray = Object.values(PetalsModels) -const voyageAIModelsArray = Object.values(VoyageAIModels) -const alephAlphaModelsArray = Object.values(AlephAlphaModels) - -export const allTextCompletionModels: LLMModels[] = [ - ...openAIChatCompletionModelsArray, - ...openAITextCompletionInstructModelsArray, - ...openAIVisionModelsArray, - ...huggingFaceModelsWithPromptFormattingArray, - ...ollamaModelsArray, - ...ollamaVisionModelsArray, - ...vertexAIGoogleModelsArray, - ...palmModelsArray, - ...googleAIStudioModelsArray, - ...mistralAIModelsArray, - ...anthropicModelsArray, - ...sageMakerModelsArray, - ...bedrockModelsArray, - ...perplexityAIModelsArray, - ...vllmModelsArray, - ...xinferenceModelsArray, - ...cloudflareWorkersAIModelsArray, - ...ai21ModelsArray, - ...nlpCloudModelsArray, - ...deepInfraChatModelsArray, - ...togetherAIModelsArray, - ...baseTenModelsArray, - ...petalsModelsArray, - ...voyageAIModelsArray, - ...alephAlphaModelsArray, -] - -// TODO: Filter these out by provided environment variables so we only show what we have enabled. -export const allModels: (LLMModels | EmbeddingModels)[] = [ - ...allTextCompletionModels, - ...allEmbeddingModels, -] diff --git a/plugins/core/src/lib/services/coreLLMService/constants/allCompletionModels.ts b/plugins/core/src/lib/services/coreLLMService/constants/allCompletionModels.ts new file mode 100644 index 0000000000..a416191570 --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/constants/allCompletionModels.ts @@ -0,0 +1,56 @@ +import { CompletionModels } from '../types/completionModels' +import { + openAIChatCompletionModelsArray, + openAITextCompletionInstructModelsArray, + openAIVisionModelsArray, + huggingFaceModelsWithPromptFormattingArray, + ollamaModelsArray, + ollamaVisionModelsArray, + vertexAIGoogleModelsArray, + palmModelsArray, + googleAIStudioModelsArray, + mistralAIModelsArray, + anthropicModelsArray, + sageMakerModelsArray, + bedrockModelsArray, + perplexityAIModelsArray, + vllmModelsArray, + xinferenceModelsArray, + cloudflareWorkersAIModelsArray, + ai21ModelsArray, + nlpCloudModelsArray, + deepInfraChatModelsArray, + togetherAIModelsArray, + baseTenModelsArray, + petalsModelsArray, + voyageAIModelsArray, + alephAlphaModelsArray, +} from './completionModelArrays' + +export const allCompletionModels: CompletionModels[] = [ + ...openAIChatCompletionModelsArray, + ...openAITextCompletionInstructModelsArray, + ...openAIVisionModelsArray, + ...huggingFaceModelsWithPromptFormattingArray, + ...ollamaModelsArray, + ...ollamaVisionModelsArray, + ...vertexAIGoogleModelsArray, + ...palmModelsArray, + ...googleAIStudioModelsArray, + ...mistralAIModelsArray, + ...anthropicModelsArray, + ...sageMakerModelsArray, + ...bedrockModelsArray, + ...perplexityAIModelsArray, + ...vllmModelsArray, + ...xinferenceModelsArray, + ...cloudflareWorkersAIModelsArray, + ...ai21ModelsArray, + ...nlpCloudModelsArray, + ...deepInfraChatModelsArray, + ...togetherAIModelsArray, + ...baseTenModelsArray, + ...petalsModelsArray, + ...voyageAIModelsArray, + ...alephAlphaModelsArray, +] diff --git a/plugins/core/src/lib/services/coreLLMService/constants/allEmbeddingModels.ts b/plugins/core/src/lib/services/coreLLMService/constants/allEmbeddingModels.ts new file mode 100644 index 0000000000..df7dbdfd80 --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/constants/allEmbeddingModels.ts @@ -0,0 +1,18 @@ +import { EmbeddingModels } from '../../coreEmbeddingService/types' +import { + bedrockEmbeddingModelsArray, + cohereEmbeddingModelsArray, + huggingFaceEmbeddingModelsArray, + mistralEmbeddingModelsArray, + openAIEmbeddingModelsArray, + voyageEmbeddingModelsArray, +} from './embeddingModelArrays' + +export const allEmbeddingModels: EmbeddingModels[] = [ + ...bedrockEmbeddingModelsArray, + ...cohereEmbeddingModelsArray, + ...mistralEmbeddingModelsArray, + ...voyageEmbeddingModelsArray, + ...openAIEmbeddingModelsArray, + ...huggingFaceEmbeddingModelsArray, +] diff --git a/plugins/core/src/lib/services/coreLLMService/constants/completionModelArrays.ts b/plugins/core/src/lib/services/coreLLMService/constants/completionModelArrays.ts new file mode 100644 index 0000000000..e7c38215ca --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/constants/completionModelArrays.ts @@ -0,0 +1,73 @@ +import { + OpenAIChatCompletionModels, + OpenAITextCompletionInstructModels, + OpenAIVisionModels, + HuggingFaceModelsWithPromptFormatting, + OllamaModels, + OllamaVisionModels, + VertexAIGoogleModels, + PalmModels, + GoogleAIStudioModels, + MistralAIModels, + AnthropicModels, + SageMakerModels, + BedrockModels, + PerplexityAIModels, + VLLMModels, + XinferenceModels, + CloudflareWorkersAIModels, + AI21Models, + NLPCloudModels, + DeepInfraChatModels, + TogetherAIModels, + BaseTenModels, + PetalsModels, + VoyageAIModels, + AlephAlphaModels, + AnyscaleModels, + OpenRouterModels, + ReplicateModels, +} from '../types/completionModels' + +export const openAIChatCompletionModelsArray = Object.values( + OpenAIChatCompletionModels +) +export const openAITextCompletionInstructModelsArray = Object.values( + OpenAITextCompletionInstructModels +) + +export const allOpenAICompletionModelsArray = [ + ...openAIChatCompletionModelsArray, + ...openAITextCompletionInstructModelsArray, +] + +export const openAIVisionModelsArray = Object.values(OpenAIVisionModels) +export const huggingFaceModelsWithPromptFormattingArray = Object.values( + HuggingFaceModelsWithPromptFormatting +) +export const ollamaModelsArray = Object.values(OllamaModels) +export const ollamaVisionModelsArray = Object.values(OllamaVisionModels) +export const vertexAIGoogleModelsArray = Object.values(VertexAIGoogleModels) +export const palmModelsArray = Object.values(PalmModels) +export const googleAIStudioModelsArray = Object.values(GoogleAIStudioModels) +export const mistralAIModelsArray = Object.values(MistralAIModels) +export const anthropicModelsArray = Object.values(AnthropicModels) +export const sageMakerModelsArray = Object.values(SageMakerModels) +export const bedrockModelsArray = Object.values(BedrockModels) +export const perplexityAIModelsArray = Object.values(PerplexityAIModels) +export const vllmModelsArray = Object.values(VLLMModels) +export const xinferenceModelsArray = Object.values(XinferenceModels) +export const cloudflareWorkersAIModelsArray = Object.values( + CloudflareWorkersAIModels +) +export const ai21ModelsArray = Object.values(AI21Models) +export const nlpCloudModelsArray = Object.values(NLPCloudModels) +export const deepInfraChatModelsArray = Object.values(DeepInfraChatModels) +export const togetherAIModelsArray = Object.values(TogetherAIModels) +export const baseTenModelsArray = Object.values(BaseTenModels) +export const petalsModelsArray = Object.values(PetalsModels) +export const voyageAIModelsArray = Object.values(VoyageAIModels) +export const alephAlphaModelsArray = Object.values(AlephAlphaModels) +export const anyscaleModelsArray = Object.values(AnyscaleModels) +export const openRouterModelsArray = Object.values(OpenRouterModels) +export const replicateModelsArray = Object.values(ReplicateModels) diff --git a/plugins/core/src/lib/services/coreLLMService/constants/embeddingModelArrays.ts b/plugins/core/src/lib/services/coreLLMService/constants/embeddingModelArrays.ts new file mode 100644 index 0000000000..f7eeafb95e --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/constants/embeddingModelArrays.ts @@ -0,0 +1,17 @@ +import { + BedrockEmbeddingModels, + CohereEmbeddingModels, + MistralEmbeddingModels, + VoyageEmbeddingModels, + OpenAIEmbeddingModels, + HuggingFaceEmbeddingModels, +} from '../../coreEmbeddingService/types' + +export const bedrockEmbeddingModelsArray = Object.values(BedrockEmbeddingModels) +export const cohereEmbeddingModelsArray = Object.values(CohereEmbeddingModels) +export const mistralEmbeddingModelsArray = Object.values(MistralEmbeddingModels) +export const voyageEmbeddingModelsArray = Object.values(VoyageEmbeddingModels) +export const openAIEmbeddingModelsArray = Object.values(OpenAIEmbeddingModels) +export const huggingFaceEmbeddingModelsArray = Object.values( + HuggingFaceEmbeddingModels +) diff --git a/plugins/core/src/lib/services/coreLLMService/constants/modelProviderMap.ts b/plugins/core/src/lib/services/coreLLMService/constants/modelProviderMap.ts new file mode 100644 index 0000000000..2f19352046 --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/constants/modelProviderMap.ts @@ -0,0 +1,834 @@ +import { + BedrockEmbeddingModels, + CohereEmbeddingModels, + MistralEmbeddingModels, + VoyageEmbeddingModels, + OpenAIEmbeddingModels, + HuggingFaceEmbeddingModels, +} from '../../coreEmbeddingService/types' +import { + OpenAIChatCompletionModels, + OpenAITextCompletionInstructModels, + OpenAIVisionModels, + HuggingFaceModelsWithPromptFormatting, + OllamaModels, + OllamaVisionModels, + VertexAIGoogleModels, + PalmModels, + MistralAIModels, + AnthropicModels, + SageMakerModels, + BedrockModels, + PerplexityAIModels, + VLLMModels, + XinferenceModels, + CloudflareWorkersAIModels, + AI21Models, + NLPCloudModels, + DeepInfraChatModels, + TogetherAIModels, + BaseTenModels, + PetalsModels, + VoyageAIModels, + AlephAlphaModels, + AnyscaleModels, + OpenRouterModels, +} from '../types/completionModels' +import { AllModels } from '../types/models' +import { + ModelProviderMapping, + LLMProviders, + LLMProviderKeys, +} from '../types/providerTypes' + +export const modelProviderMap: Record = { + /// OpenAI Chat Completion Models + [OpenAIChatCompletionModels.GPT35Turbo1106Preview]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAIChatCompletionModels.GPT35Turbo]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAIChatCompletionModels.GPT35Turbo1106]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAIChatCompletionModels.GPT35Turbo0301]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAIChatCompletionModels.GPT35Turbo0613]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAIChatCompletionModels.GPT35Turbo16k]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAIChatCompletionModels.GPT35Turbo16k0613]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAIChatCompletionModels.GPT4]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAIChatCompletionModels.GPT40314]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAIChatCompletionModels.GPT40613]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + + // OpenAI Text Completion / Instruct Models + [OpenAITextCompletionInstructModels.Ada001]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAITextCompletionInstructModels.Babbage001]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAITextCompletionInstructModels.Babbage002]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAITextCompletionInstructModels.Curie001]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAITextCompletionInstructModels.Davinci002]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAITextCompletionInstructModels.GPT35TurboInstruct]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [OpenAITextCompletionInstructModels.TextDavinci003]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + + // OpenAI Vision Models + [OpenAIVisionModels.GPT4VisionPreview]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + + // HuggingFace Models With Prompt Formatting + [HuggingFaceModelsWithPromptFormatting.HuggingFaceCodellamaCodeLlama34bInstructHf]: + { + provider: LLMProviders.HuggingFace, + apiKeyName: LLMProviderKeys.HuggingFace, + }, + [HuggingFaceModelsWithPromptFormatting.HuggingFaceMetaLlamaLlama27bChat]: { + provider: LLMProviders.HuggingFace, + apiKeyName: LLMProviderKeys.HuggingFace, + }, + [HuggingFaceModelsWithPromptFormatting.HuggingFaceMistral7BInstructV01]: { + provider: LLMProviders.HuggingFace, + apiKeyName: LLMProviderKeys.HuggingFace, + }, + [HuggingFaceModelsWithPromptFormatting.HuggingFaceMosaicmlMpt7bChat]: { + provider: LLMProviders.HuggingFace, + apiKeyName: LLMProviderKeys.HuggingFace, + }, + [HuggingFaceModelsWithPromptFormatting.HuggingFacePhindPhindCodeLlama34Bv2]: { + provider: LLMProviders.HuggingFace, + apiKeyName: LLMProviderKeys.HuggingFace, + }, + [HuggingFaceModelsWithPromptFormatting.HuggingFaceTiiuaeFalcon7bInstruct]: { + provider: LLMProviders.HuggingFace, + apiKeyName: LLMProviderKeys.HuggingFace, + }, + [HuggingFaceModelsWithPromptFormatting.HuggingFaceWizardLMWizardCoderPython34BV10]: + { + provider: LLMProviders.HuggingFace, + apiKeyName: LLMProviderKeys.HuggingFace, + }, + + // Ollama Models + [OllamaModels.OlamaMistral]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + [OllamaModels.OlamaLlama27B]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + [OllamaModels.OlamaLlama213B]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + [OllamaModels.OlamaLlama270B]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + [OllamaModels.OlamaLlama2Uncensored]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + [OllamaModels.OlamaCodeLlama]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + [OllamaModels.OlamaNousHermes]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + [OllamaModels.OlamaOrcaMini]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + [OllamaModels.OlamaVicuna]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + [OllamaModels.OlamaNousHermes13B]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + [OllamaModels.OlamaWizardVicunaUncensored]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + + // Ollama Vision Models + [OllamaVisionModels.LLAVA]: { + provider: LLMProviders.Ollama, + apiKeyName: LLMProviderKeys.Ollama, + }, + + // VertexAI + [VertexAIGoogleModels.GeminiPro]: { + provider: LLMProviders.VertexAI, + apiKeyName: LLMProviderKeys.VertexAI, + }, + [VertexAIGoogleModels.GeminiProVision]: { + provider: LLMProviders.VertexAI, + apiKeyName: LLMProviderKeys.VertexAI, + }, + + //Palm + [PalmModels.ChatBison]: { + provider: LLMProviders.Palm, + apiKeyName: LLMProviderKeys.Palm, + }, + + // // Google AI Studio + // [GoogleAIStudioModels.GeminiPro]: { + // provider: LLMProviders.GoogleAIStudio, + // apiKeyName: LLMProviderKeys.GoogleAIStudio, + // }, + // [GoogleAIStudioModels.GeminiProVision]: { + // provider: LLMProviders.GoogleAIStudio, + // apiKeyName: LLMProviderKeys.GoogleAIStudio, + // }, + + // Mistral AI Models + [MistralAIModels.MistralTiny]: { + provider: LLMProviders.Mistral, + apiKeyName: LLMProviderKeys.Mistral, + }, + [MistralAIModels.MistralSmall]: { + provider: LLMProviders.Mistral, + apiKeyName: LLMProviderKeys.Mistral, + }, + [MistralAIModels.MistralMedium]: { + provider: LLMProviders.Mistral, + apiKeyName: LLMProviderKeys.Mistral, + }, + + // Anthropic Models + [AnthropicModels.Claude21]: { + provider: LLMProviders.Anthropic, + apiKeyName: LLMProviderKeys.Anthropic, + }, + [AnthropicModels.Claude2]: { + provider: LLMProviders.Anthropic, + apiKeyName: LLMProviderKeys.Anthropic, + }, + [AnthropicModels.ClaudeInstant1]: { + provider: LLMProviders.Anthropic, + apiKeyName: LLMProviderKeys.Anthropic, + }, + [AnthropicModels.ClaudeInstant12]: { + provider: LLMProviders.Anthropic, + apiKeyName: LLMProviderKeys.Anthropic, + }, + + // SageMaker Models + [SageMakerModels.MetaLlama213B]: { + provider: LLMProviders.Sagemaker, + apiKeyName: LLMProviderKeys.Sagemaker, + }, + [SageMakerModels.MetaLlama213BChatFineTuned]: { + provider: LLMProviders.Sagemaker, + apiKeyName: LLMProviderKeys.Sagemaker, + }, + [SageMakerModels.MetaLlama270B]: { + provider: LLMProviders.Sagemaker, + apiKeyName: LLMProviderKeys.Sagemaker, + }, + [SageMakerModels.MetaLlama270BChatFineTuned]: { + provider: LLMProviders.Sagemaker, + apiKeyName: LLMProviderKeys.Sagemaker, + }, + [SageMakerModels.MetaLlama27B]: { + provider: LLMProviders.Sagemaker, + apiKeyName: LLMProviderKeys.Sagemaker, + }, + [SageMakerModels.MetaLlama27BChatFineTuned]: { + provider: LLMProviders.Sagemaker, + apiKeyName: LLMProviderKeys.Sagemaker, + }, + + // Bedrock Models + [BedrockModels.BedrockAI21J2Mid]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockModels.BedrockAI21J2Ultra]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockModels.BedrockAmazonTitanExpress]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockModels.BedrockAnthropicClaudeInstantV1]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockModels.BedrockAnthropicClaudeV1]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockModels.BedrockAnthropicClaudeV2]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockModels.BedrockAnthropicClaudeV21]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockModels.BedrockCohereCommand]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockModels.BedrockMetaLlama2Chat13b]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockModels.BedrockMetaLlama2Chat70b]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockModels.BedrockAmazonTitanLite]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + + // Perplexity AI Models + [PerplexityAIModels.Pplx70bChat]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.Pplx70bChatAlpha]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.Pplx70bOnline]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.Pplx7bOnline]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.PplxCodeLlama34bInstruct]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.PplxLlama213bChat]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.PplxLlama270bChat]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.PplxMistral7bInstruct]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.PplxOpenhermes25Mistral7b]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.PplxOpenhermes2Mistral7b]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.Pplx7bChat]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + [PerplexityAIModels.Pplx7bChatAlpha]: { + provider: LLMProviders.PerplexityAI, + apiKeyName: LLMProviderKeys.PerplexityAI, + }, + + // VLLM Models + [VLLMModels.CodellamaCodeLlama34bInstructHf]: { + provider: LLMProviders.VLLM, + apiKeyName: LLMProviderKeys.VLLM, + }, + [VLLMModels.MetaLlamaLlama27bChat]: { + provider: LLMProviders.VLLM, + apiKeyName: LLMProviderKeys.VLLM, + }, + [VLLMModels.MosaicmlMpt7bChat]: { + provider: LLMProviders.VLLM, + apiKeyName: LLMProviderKeys.VLLM, + }, + [VLLMModels.PhindPhindCodeLlama34Bv2]: { + provider: LLMProviders.VLLM, + apiKeyName: LLMProviderKeys.VLLM, + }, + [VLLMModels.TiiuaeFalcon7bInstruct]: { + provider: LLMProviders.VLLM, + apiKeyName: LLMProviderKeys.VLLM, + }, + [VLLMModels.WizardLMWizardCoderPython34BV10]: { + provider: LLMProviders.VLLM, + apiKeyName: LLMProviderKeys.VLLM, + }, + + //Xinference Models + [XinferenceModels.BgeBaseEn]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeBaseEnV15]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeBaseZh]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeLargeEn]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeLargeEnV15]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeLargeZh]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeLargeZhNoinstruct]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeLargeZhV15]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeSmallEnV15]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeSmallZh]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeSmallZhV15]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.E5LargeV2]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.GteBase]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.GteLarge]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.JinaEmbeddingsV2BaseEn]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.JinaEmbeddingsV2SmallEn]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.MultilingualE5Large]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + [XinferenceModels.BgeBaseZhV15]: { + provider: LLMProviders.Xinference, + apiKeyName: LLMProviderKeys.Xinference, + }, + + // Cloudflare Workers AI Models + [CloudflareWorkersAIModels.MetaLlama27bChatFp16]: { + provider: LLMProviders.CloudflareWorkersAI, + apiKeyName: LLMProviderKeys.CloudflareWorkersAI, + }, + [CloudflareWorkersAIModels.MetaLlama27bChatInt8]: { + provider: LLMProviders.CloudflareWorkersAI, + apiKeyName: LLMProviderKeys.CloudflareWorkersAI, + }, + [CloudflareWorkersAIModels.Mistral7bInstructV01]: { + provider: LLMProviders.CloudflareWorkersAI, + apiKeyName: LLMProviderKeys.CloudflareWorkersAI, + }, + [CloudflareWorkersAIModels.TheBlokeCodellama7bInstructAwq]: { + provider: LLMProviders.CloudflareWorkersAI, + apiKeyName: LLMProviderKeys.CloudflareWorkersAI, + }, + + // AI21 Models + [AI21Models.J2Light]: { + provider: LLMProviders.AI21, + apiKeyName: LLMProviderKeys.AI21, + }, + [AI21Models.J2Mid]: { + provider: LLMProviders.AI21, + apiKeyName: LLMProviderKeys.AI21, + }, + [AI21Models.J2Ultra]: { + provider: LLMProviders.AI21, + apiKeyName: LLMProviderKeys.AI21, + }, + + //NLPCloud Models + [NLPCloudModels.ChatDolphin]: { + provider: LLMProviders.NLPCloud, + apiKeyName: LLMProviderKeys.NLPCloud, + }, + [NLPCloudModels.Dolphin]: { + provider: LLMProviders.NLPCloud, + apiKeyName: LLMProviderKeys.NLPCloud, + }, + + // Deep Infra Models + [DeepInfraChatModels.CodellamaCodeLlama34bInstructHf]: { + provider: LLMProviders.DeepInfra, + apiKeyName: LLMProviderKeys.DeepInfra, + }, + [DeepInfraChatModels.JondurbinAiroborosL270bGpt4141]: { + provider: LLMProviders.DeepInfra, + apiKeyName: LLMProviderKeys.DeepInfra, + }, + [DeepInfraChatModels.MetaLlamaLlama213bChatHf]: { + provider: LLMProviders.DeepInfra, + apiKeyName: LLMProviderKeys.DeepInfra, + }, + [DeepInfraChatModels.MetaLlamaLlama270bChatHf]: { + provider: LLMProviders.DeepInfra, + apiKeyName: LLMProviderKeys.DeepInfra, + }, + [DeepInfraChatModels.MetaLlamaLlama27bChatHf]: { + provider: LLMProviders.DeepInfra, + apiKeyName: LLMProviderKeys.DeepInfra, + }, + [DeepInfraChatModels.MistralaiMistral7BInstructV01]: { + provider: LLMProviders.DeepInfra, + apiKeyName: LLMProviderKeys.DeepInfra, + }, + + // Together AI Models + [TogetherAIModels.TogetherLlama270bChat]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherLlama270b]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherLlama27B32K]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherLlama27B32KInstruct]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherLlama27b]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherFalcon40bInstruct]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherFalcon7bInstruct]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherAlpaca7b]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherStarchatAlpha]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherCodeLlama34b]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherCodeLlama34bInstruct]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherCodeLlama34bPython]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherSqlCoder]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherNSQLLlama27B]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherWizardCoder15BV10]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherNousHermesLlama213b]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherChronosHermes13b]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherSolar070b16bit]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + [TogetherAIModels.TogetherWizardLM70BV10]: { + provider: LLMProviders.TogetherAI, + apiKeyName: LLMProviderKeys.TogetherAI, + }, + + // Base Ten Models + [BaseTenModels.Falcon7B]: { + provider: LLMProviders.Baseten, + apiKeyName: LLMProviderKeys.Baseten, + }, + [BaseTenModels.MPT7BBase]: { + provider: LLMProviders.Baseten, + apiKeyName: LLMProviderKeys.Baseten, + }, + [BaseTenModels.WizardLM]: { + provider: LLMProviders.Baseten, + apiKeyName: LLMProviderKeys.Baseten, + }, + + // Petals Models + [PetalsModels.HuggyLlamaLlama65]: { + provider: LLMProviders.Petals, + apiKeyName: LLMProviderKeys.Petals, + }, + [PetalsModels.PetalsTeamStableBeluga]: { + provider: LLMProviders.Petals, + apiKeyName: LLMProviderKeys.Petals, + }, + + // Voyage AI Models + [VoyageAIModels.Voyage01]: { + provider: LLMProviders.VoyageAI, + apiKeyName: LLMProviderKeys.VoyageAI, + }, + [VoyageAIModels.VoyageLite01]: { + provider: LLMProviders.VoyageAI, + apiKeyName: LLMProviderKeys.VoyageAI, + }, + [VoyageAIModels.VoyageLite01Instruct]: { + provider: LLMProviders.VoyageAI, + apiKeyName: LLMProviderKeys.VoyageAI, + }, + + // Aleph Alpha Models + [AlephAlphaModels.LuminousBase]: { + provider: LLMProviders.AlephAlpha, + apiKeyName: LLMProviderKeys.AlephAlpha, + }, + [AlephAlphaModels.LuminousBaseControl]: { + provider: LLMProviders.AlephAlpha, + apiKeyName: LLMProviderKeys.AlephAlpha, + }, + [AlephAlphaModels.LuminousExtended]: { + provider: LLMProviders.AlephAlpha, + apiKeyName: LLMProviderKeys.AlephAlpha, + }, + [AlephAlphaModels.LuminousExtendedControl]: { + provider: LLMProviders.AlephAlpha, + apiKeyName: LLMProviderKeys.AlephAlpha, + }, + [AlephAlphaModels.LuminousSupreme]: { + provider: LLMProviders.AlephAlpha, + apiKeyName: LLMProviderKeys.AlephAlpha, + }, + [AlephAlphaModels.LuminousSupremeControl]: { + provider: LLMProviders.AlephAlpha, + apiKeyName: LLMProviderKeys.AlephAlpha, + }, + + // Embedding Models + [BedrockEmbeddingModels.AmazonTitanEmbedTextV1]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockEmbeddingModels.CohereEmbedEnglishV3]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + [BedrockEmbeddingModels.CohereEmbedMultilingualV3]: { + provider: LLMProviders.Bedrock, + apiKeyName: LLMProviderKeys.Bedrock, + }, + + [CohereEmbeddingModels.EmbedEnglishLightV20]: { + provider: LLMProviders.Cohere, + apiKeyName: LLMProviderKeys.Cohere, + }, + [CohereEmbeddingModels.EmbedEnglishLightV30]: { + provider: LLMProviders.Cohere, + apiKeyName: LLMProviderKeys.Cohere, + }, + [CohereEmbeddingModels.EmbedEnglishV20]: { + provider: LLMProviders.Cohere, + apiKeyName: LLMProviderKeys.Cohere, + }, + [CohereEmbeddingModels.EmbedEnglishV30]: { + provider: LLMProviders.Cohere, + apiKeyName: LLMProviderKeys.Cohere, + }, + [CohereEmbeddingModels.EmbedMultilingualLightV30]: { + provider: LLMProviders.Cohere, + apiKeyName: LLMProviderKeys.Cohere, + }, + [CohereEmbeddingModels.EmbedMultilingualV20]: { + provider: LLMProviders.Cohere, + apiKeyName: LLMProviderKeys.Cohere, + }, + [CohereEmbeddingModels.EmbedMultilingualV30]: { + provider: LLMProviders.Cohere, + apiKeyName: LLMProviderKeys.Cohere, + }, + + [MistralEmbeddingModels.MistralEmbed]: { + provider: LLMProviders.Mistral, + apiKeyName: LLMProviderKeys.Mistral, + }, + + [VoyageEmbeddingModels.Voyage02]: { + provider: LLMProviders.VoyageAI, + apiKeyName: LLMProviderKeys.VoyageAI, + }, + [VoyageEmbeddingModels.VoyageCode02]: { + provider: LLMProviders.VoyageAI, + apiKeyName: LLMProviderKeys.VoyageAI, + }, + [OpenAIEmbeddingModels.TextEmbeddingAda002]: { + provider: LLMProviders.OpenAI, + apiKeyName: LLMProviderKeys.OpenAI, + }, + [HuggingFaceEmbeddingModels.HuggingFaceAnyHfEmbeddingModel]: { + provider: LLMProviders.HuggingFace, + apiKeyName: LLMProviderKeys.HuggingFace, + }, + [HuggingFaceEmbeddingModels.HuggingFaceBAAIBgeLargeZh]: { + provider: LLMProviders.HuggingFace, + apiKeyName: LLMProviderKeys.HuggingFace, + }, + [HuggingFaceEmbeddingModels.HuggingFaceMicrosoftCodebertBase]: { + provider: LLMProviders.HuggingFace, + apiKeyName: LLMProviderKeys.HuggingFace, + }, + [AnyscaleModels.AnyscaleCodellamaCodeLlama34bInstructHf]: { + provider: LLMProviders.Anyscale, + apiKeyName: LLMProviderKeys.Anyscale, + }, + + [AnyscaleModels.AnyscaleMetaLlamaLlama213bChatHf]: { + provider: LLMProviders.Anyscale, + apiKeyName: LLMProviderKeys.Anyscale, + }, + [AnyscaleModels.AnyscaleMetaLlamaLlama270bChatHf]: { + provider: LLMProviders.Anyscale, + apiKeyName: LLMProviderKeys.Anyscale, + }, + [AnyscaleModels.AnyscaleMetaLlamaLlama27bChatHf]: { + provider: LLMProviders.Anyscale, + apiKeyName: LLMProviderKeys.Anyscale, + }, + [AnyscaleModels.AnyscaleZephyr7BBeta]: { + provider: LLMProviders.Anyscale, + apiKeyName: LLMProviderKeys.Anyscale, + }, + [AnyscaleModels.AnyscaleMistral7BInstructV01]: { + provider: LLMProviders.Anyscale, + apiKeyName: LLMProviderKeys.Anyscale, + }, + [OpenRouterModels.OpenRouterAnthropicClaud2]: { + provider: LLMProviders.OpenRouter, + apiKeyName: LLMProviderKeys.OpenRouter, + }, + [OpenRouterModels.OpenRouterAnthropicClaudInstantV1]: { + provider: LLMProviders.OpenRouter, + apiKeyName: LLMProviderKeys.OpenRouter, + }, + [OpenRouterModels.OpenRouterMetaLlamaLlama213bChat]: { + provider: LLMProviders.OpenRouter, + apiKeyName: LLMProviderKeys.OpenRouter, + }, + [OpenRouterModels.OpenRouterMetaLlamaLlama270bChat]: { + provider: LLMProviders.OpenRouter, + apiKeyName: LLMProviderKeys.OpenRouter, + }, + [OpenRouterModels.OpenRouterOpenAIGpt35turbo]: { + provider: LLMProviders.OpenRouter, + apiKeyName: LLMProviderKeys.OpenRouter, + }, + [OpenRouterModels.OpenRouterOpenAIGpt35turbo16k]: { + provider: LLMProviders.OpenRouter, + apiKeyName: LLMProviderKeys.OpenRouter, + }, + [OpenRouterModels.OpenRouterOpenAIGpt4]: { + provider: LLMProviders.OpenRouter, + apiKeyName: LLMProviderKeys.OpenRouter, + }, + [OpenRouterModels.OpenRouterOpenAIGpt432k]: { + provider: LLMProviders.OpenRouter, + apiKeyName: LLMProviderKeys.OpenRouter, + }, + [OpenRouterModels.OpenRouterPalm2ChatBison]: { + provider: LLMProviders.OpenRouter, + apiKeyName: LLMProviderKeys.OpenRouter, + }, + [OpenRouterModels.OpenRouterPalm2CodeChatBison]: { + provider: LLMProviders.OpenRouter, + apiKeyName: LLMProviderKeys.OpenRouter, + }, +} diff --git a/plugins/core/src/lib/services/coreLLMService/constants/providerModelMap.ts b/plugins/core/src/lib/services/coreLLMService/constants/providerModelMap.ts new file mode 100644 index 0000000000..f9f6af0c0c --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/constants/providerModelMap.ts @@ -0,0 +1,22 @@ +import { AllModels } from '../types/models' +import { LLMProviders } from '../types/providerTypes' +import { modelProviderMap } from './modelProviderMap' + +type ProviderModelMapping = { + [provider in LLMProviders]?: AllModels[] +} + +export const providerModelMap: ProviderModelMapping = {} + +// Iterate over each model in modelProviderMap +Object.entries(modelProviderMap).forEach(([model, providerMapping]) => { + const provider = providerMapping.provider + + // Initialize an array for the provider if it doesn't exist + if (!providerModelMap[provider]) { + providerModelMap[provider] = [] + } + + // Add the model to the provider's array + providerModelMap[provider]?.push(model as AllModels) +}) diff --git a/plugins/core/src/lib/services/coreLLMService/constants/providers.ts b/plugins/core/src/lib/services/coreLLMService/constants/providers.ts new file mode 100644 index 0000000000..ea302dd231 --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/constants/providers.ts @@ -0,0 +1,14 @@ +import { LLMProviders } from '../types/providerTypes' + +export type ActiveProviders = + | LLMProviders.OpenAI + | LLMProviders.GoogleAIStudio + | LLMProviders.TogetherAI +// | LLMProviders.VertexAI + +export const activeProviders: ActiveProviders[] = [ + LLMProviders.OpenAI, + LLMProviders.TogetherAI, + LLMProviders.GoogleAIStudio, + // LLMProviders.VertexAI, +] diff --git a/plugins/core/src/lib/services/coreLLMService/coreLLMService.ts b/plugins/core/src/lib/services/coreLLMService/coreLLMService.ts index bd26c293c2..1d5b6c5268 100644 --- a/plugins/core/src/lib/services/coreLLMService/coreLLMService.ts +++ b/plugins/core/src/lib/services/coreLLMService/coreLLMService.ts @@ -1,22 +1,18 @@ import { python } from 'pythonia' -import { - NODE_ENV, - PRODUCTION, - VERTEXAI_LOCATION, - VERTEXAI_PROJECT, -} from 'shared/config' +import { PRODUCTION } from 'shared/config' import { - LLMCredential, - LLMProviderKeys, - LLMModels, - CompletionResponse, -} from './types' -import { modelProviderMap } from './constants' -import { ICoreBudgetManagerService, ICoreLLMService } from '../types' + // CompletionParams, + ICoreBudgetManagerService, + ICoreLLMService, +} from '../types' import { CoreBudgetManagerService } from '../coreBudgetManagerService/coreBudgetMangerService' import { UserService } from '../userService/userService' import { saveRequest } from 'shared/core' +import { findProviderKey, findProviderName } from './findProvider' +import { LLMCredential } from './types/providerTypes' +import { CompletionResponse } from './types/completionTypes' +import { AllModels } from './types/models' const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) @@ -41,8 +37,6 @@ export class CoreLLMService implements ICoreLLMService { async initialize() { try { this.liteLLM = await python('litellm') - this.liteLLM.vertex_project = VERTEXAI_PROJECT - this.liteLLM.vertex_location = VERTEXAI_LOCATION this.liteLLM.set_verbose = true this.liteLLM.drop_params = true this.coreBudgetManagerService = new CoreBudgetManagerService( @@ -139,7 +133,7 @@ export class CoreLLMService implements ICoreLLMService { status: '', statusCode: 200, parameters: JSON.stringify(request.options), - provider: this.findProvider(request.model), + provider: findProviderName(request.model), type: 'completion', hidden: false, processed: false, @@ -177,26 +171,25 @@ export class CoreLLMService implements ICoreLLMService { } } - private findProvider = (model: LLMModels): LLMProviderKeys => { - return modelProviderMap[model] - } - - private getCredential = (model: LLMModels): string => { - const provider = this.findProvider(model) + private getCredential = (model: AllModels): string => { + const providerKey = findProviderKey(model) - let credential = this.credentials.find(c => c.name === provider)?.value + let credential = this.credentials.find(c => c.name === providerKey)?.value - if (!credential && PRODUCTION) { - credential = process.env[provider] - } - - if (!credential && NODE_ENV === 'development' && !PRODUCTION) { - credential = process.env[provider] + if (!credential && !PRODUCTION && providerKey) { + credential = process.env[providerKey] } if (!credential) { - throw new Error(`No credential found for ${provider}`) + throw new Error(`No credential found for ${providerKey}`) } return credential } + + // getProvidersWithCredentials(): LLMProviders[] { + // const credentialsArray = this.credentials + // const credentials = credentialsArray.map(cred => cred.name) as any + // const providers = getProvidersWithUserKeys(credentials) + // return providers + // } } diff --git a/plugins/core/src/lib/services/coreLLMService/findProvider.ts b/plugins/core/src/lib/services/coreLLMService/findProvider.ts new file mode 100644 index 0000000000..41e61e07bd --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/findProvider.ts @@ -0,0 +1,33 @@ +import { modelProviderMap } from './constants/modelProviderMap' +import { providerModelMap } from './constants/providerModelMap' +import { AllModels } from './types/models' +import { LLMProviderKeys, LLMProviders } from './types/providerTypes' + +export function findProviderKey(model: AllModels): LLMProviderKeys | undefined { + return modelProviderMap[model].apiKeyName +} + +export function findProviderName(model: AllModels): LLMProviders | undefined { + return modelProviderMap[model].provider +} + +export function getModelsForProvider( + provider: LLMProviders +): Partial { + return providerModelMap[provider] +} + +export function getProvidersWithUserKeys( + keys: LLMProviderKeys[] +): LLMProviders[] { + return keys + .map(key => { + const providerName = Object.keys(LLMProviderKeys).find( + name => LLMProviderKeys[name] === key + ) + return providerName + ? (providerName.replace('_API_KEY', '') as LLMProviders) + : undefined + }) + .filter(provider => provider !== undefined) as LLMProviders[] +} diff --git a/plugins/core/src/lib/services/coreLLMService/types.ts b/plugins/core/src/lib/services/coreLLMService/types/completionModels.ts similarity index 50% rename from plugins/core/src/lib/services/coreLLMService/types.ts rename to plugins/core/src/lib/services/coreLLMService/types/completionModels.ts index 0234f0d5a2..db8c26190d 100644 --- a/plugins/core/src/lib/services/coreLLMService/types.ts +++ b/plugins/core/src/lib/services/coreLLMService/types/completionModels.ts @@ -1,180 +1,9 @@ -import { type PluginCredential } from 'server/credentials' -import { EmbeddingModels } from '../coreEmbeddingService/types' - -export type FunctionType = { - name: string - description?: string - parameters: Record -} - -export type Message = { - role: string - content: string | null - name?: string - function_call?: { - type: string - function: FunctionType - } -} - -export type LiteLLMOptions = { - api_base?: string - api_version?: string - api_key?: string - num_retries?: number - context_window_fallback_dict?: Record - fallbacks?: Array> - metadata?: Record -} - -export type ToolType = { - type: string - function: FunctionType -} - -export type CompletionOptions = { - temperature?: number - top_p?: number - n?: number - stream?: boolean - stop?: string | string[] - max_tokens?: number - presence_penalty?: number - frequency_penalty?: number - logit_bias?: Record - user?: string - deployment_id?: string - request_timeout?: number - response_format?: { - type: string - } - seed?: number - tools?: ToolType[] - tool_choice?: string | ToolType - functions?: FunctionType[] - function_call?: string -} & LiteLLMOptions - -export type CompletionRequest = { - model: LLMModels - messages: Message[] - options?: CompletionOptions - api_key?: string -} - -export type Chunk = { - choices: ChunkChoice[] -} - -export type ChunkChoice = { - finish_reason: string - index: number - delta: { - function_call: string - tool_calls: string - content: string - role: string - } -} - -export type Choice = { - finish_reason: string - index: number - message: { - role: string - content: string - } -} - -export type CompletionResponse = { - id: string - choices: Choice[] - created: string - model: string - object: string - system_fingerprint: any | null - usage: { - prompt_tokens: number - completion_tokens: number - total_tokens: number - } - _python_object: any -} - -export enum LLMProviderKeys { - OpenAI = 'OPENAI_API_KEY', - Azure = 'AZURE_API_KEY', - Anthropic = 'ANTHROPIC_API_KEY', - Sagemaker = 'SAGEMAKER_API_KEY', - Bedrock = 'BEDROCK_API_KEY', - Anyscale = 'ANYSCALE_API_KEY', - PerplexityAI = 'PERPLEXITY_API_KEY', - VLLM = 'VLLM_API_KEY', - DeepInfra = 'DEEPINFRA_API_KEY', - Cohere = 'COHERE_API_KEY', - TogetherAI = 'TOGETHER_AI_API_KEY', - AlephAlpha = 'ALEPH_ALPHA_API_KEY', - Baseten = 'BASETEN_API_KEY', - OpenRouter = 'OPENROUTER_API_KEY', - CustomAPI = 'CUSTOM_API_KEY', - CustomOpenAI = 'CUSTOM_OPEN_API_KEY', - Petals = 'PETALS_API_KEY', - Ollama = 'OLLAMA_API_KEY', - GoogleAIStudio = 'GOOGLEAISTUDIO_API_KEY', - Palm = 'PALM_API_KEY', - HuggingFace = 'HUGGINGFACE_API_KEY', - Xinference = 'XINFERENCE_API_KEY', - CloudflareWorkersAI = 'CLOUDFLAREWORKERSAI_API_KEY', - AI21 = 'AI21_API_KEY', - NLPCloud = 'NLPCLOUD_API_KEY', - VoyageAI = 'VOYAGEAI_API_KEY', - Replicate = 'REPLICATE_API_KEY', - Meta = 'META_API_KEY', - Mistral = 'MISTRAL_API_KEY', - VertexAI = 'VERTEXAI_API_KEY', - Unknown = 'unknown', -} - -export enum LLMProviders { - OpenAI = 'OpenAI', - Azure = 'Azure', - Anthropic = 'Anthropic', - Sagemaker = 'Sagemaker', - Bedrock = 'Bedrock', - Anyscale = 'Anyscale', - PerplexityAI = 'PerplexityAI', - VLLM = 'VLLM', - DeepInfra = 'DeepInfra', - Cohere = 'Cohere', - TogetherAI = 'TogetherAI', - AlephAlpha = 'AlephAlpha', - Baseten = 'Baseten', - OpenRouter = 'OpenRouter', - CustomAPI = 'CustomAPI', - CustomOpenAI = 'CustomOpenAI', - Petals = 'Petals', - Ollama = 'Ollama', - GoogleAIStudio = 'GoogleAIStudio', - Palm = 'Palm', - HuggingFace = 'HuggingFace', - Xinference = 'Xinference', - CloudflareWorkersAI = 'CloudflareWorkersAI', - AI21 = 'AI21', - NLPCloud = 'NLPCloud', - VoyageAI = 'VoyageAI', - Replicate = 'Replicate', - Meta = 'Meta', - Mistral = 'Mistral', - VertexAI = 'VertexAI', -} - -export type Models = EmbeddingModels | LLMModels - -export type LLMCredential = PluginCredential & { - value: string -} - -export type LLMModel = [LLMModels, LLMProviderKeys] +import { + BedrockEmbeddingModels, + HuggingFaceEmbeddingModels, + MistralEmbeddingModels, + VoyageEmbeddingModels, +} from '../../coreEmbeddingService/types' export enum OpenAIChatCompletionModels { GPT35Turbo1106Preview = 'gpt-4-1106-preview', @@ -203,35 +32,27 @@ export enum OpenAITextCompletionInstructModels { Davinci002 = 'davinci-002', } -export type AzureModel = `azure/${string}` -export type CustomOpenAIModel = `openai/${string}` - -// export enum AzureOpenAIChatCompletionModels { -// Gpt4 = 'azure/gpt-4', -// Gpt40314 = 'azure/gpt-4-0314', -// Gpt40613 = 'azure/gpt-4-0613', -// Gpt432k = 'azure/gpt-4-32k', -// Gpt432k0314 = 'azure/gpt-4-32k-0314', -// Gpt432k0613 = 'azure/gpt-4-32k-0613', -// Gpt35Turbo = 'azure/gpt-3.5-turbo', -// Gpt35Turbo0301 = 'azure/gpt-3.5-turbo-0301', -// Gpt35Turbo0613 = 'azure/gpt-3.5-turbo-0613', -// Gpt35Turbo16k = 'azure/gpt-3.5-turbo-16k', -// Gpt35Turbo16k0613 = 'azure/gpt-3.5-turbo-16k-0613', -// } - -// export enum AzureOpenAIVisionModels { -// GPT4Vision = 'gpt-4-vision', -// } +export type AzureSlug = `azure/${string}` +export type CustomOpenAISlug = `openai/${string}` +export type CustomReplicateSlug = `replicate/${string}` +export type ReplicateDeploymentModelSlug = `replicate/deployments/${string}` + +export type HuggingFaceWithPromptFormattingSlug = + `huggingface/${HuggingFaceModelsWithPromptFormatting}` + +export type HuggingFaceSlug = `huggingface/${ + | HuggingFaceEmbeddingModels + | HuggingFaceWithPromptFormattingSlug + | string}` export enum HuggingFaceModelsWithPromptFormatting { - HuggingFaceMistral7BInstructV01 = 'huggingface/mistralai/Mistral-7B-Instruct-v0.1', - HuggingFaceMetaLlamaLlama27bChat = 'huggingface/meta-llama/Llama-2-7b-chat', - HuggingFaceTiiuaeFalcon7bInstruct = 'huggingface/tiiuae/falcon-7b-instruct', - HuggingFaceMosaicmlMpt7bChat = 'huggingface/mosaicml/mpt-7b-chat', - HuggingFaceCodellamaCodeLlama34bInstructHf = 'huggingface/codellama/CodeLlama-34b-Instruct-hf', - HuggingFaceWizardLMWizardCoderPython34BV10 = 'huggingface/WizardLM/WizardCoder-Python-34B-V1.0', - HuggingFacePhindPhindCodeLlama34Bv2 = 'huggingface/Phind/Phind-CodeLlama-34B-v2', + HuggingFaceMistral7BInstructV01 = 'mistralai/Mistral-7B-Instruct-v0.1', + HuggingFaceMetaLlamaLlama27bChat = 'meta-llama/Llama-2-7b-chat', + HuggingFaceTiiuaeFalcon7bInstruct = 'tiiuae/falcon-7b-instruct', + HuggingFaceMosaicmlMpt7bChat = 'mosaicml/mpt-7b-chat', + HuggingFaceCodellamaCodeLlama34bInstructHf = 'codellama/CodeLlama-34b-Instruct-hf', + HuggingFaceWizardLMWizardCoderPython34BV10 = 'WizardLM/WizardCoder-Python-34B-V1.0', + HuggingFacePhindPhindCodeLlama34Bv2 = 'Phind/Phind-CodeLlama-34B-v2', } export enum OllamaVisionModels { @@ -261,15 +82,19 @@ export enum PalmModels { ChatBison = 'chat-bison', } +export type GoogleAIStudioSlug = `gemini/${GoogleAIStudioModels}` export enum GoogleAIStudioModels { - GeminiGeminiPro = 'gemini/gemini-pro', - GeminiGeminiProVision = '/gemini/gemini-pro-vision', + GeminiPro = 'gemini-pro', + GeminiProVision = 'gemini-pro-vision', } +export type MistralAISlug = `mistral/${ + | MistralAIModels + | MistralEmbeddingModels}` export enum MistralAIModels { - MistralTiny = 'mistral/mistral-tiny', - MistralSmall = 'mistral/mistral-small', - MistralMedium = 'mistral/mistral-medium', + MistralTiny = 'mistral-tiny', + MistralSmall = 'mistral-small', + MistralMedium = 'mistral-medium', } export enum AnthropicModels { @@ -288,33 +113,35 @@ export enum SageMakerModels { MetaLlama270BChatFineTuned = 'sagemaker/jumpstart-dft-meta-textgeneration-llama-2-70b-b-f', } +export type BedrockSlug = `bedrock/${BedrockModels | BedrockEmbeddingModels}` export enum BedrockModels { - BedrockAnthropicClaudeV21 = 'bedrock/anthropic.claude-v2:1', - BedrockAnthropicClaudeV2 = 'bedrock/anthropic.claude-v2', - BedrockAnthropicClaudeInstantV1 = 'bedrock/anthropic.claude-instant-v1', - BedrockAnthropicClaudeV1 = 'bedrock/anthropic.claude-v1', - BedrockAmazonTitanLite = 'bedrock/amazon.titan-text-lite-v1', - BedrockAmazonTitanExpress = 'bedrock/amazon.titan-text-express-v1', - BedrockCohereCommand = 'bedrock/cohere.command-text-v14', - BedrockAI21J2Mid = 'bedrock/ai21.j2-mid-v1', - BedrockAI21J2Ultra = 'bedrock/ai21.j2-ultra-v1', - BedrockMetaLlama2Chat13b = 'bedrock/meta.llama2-13b-chat-v1', - BedrockMetaLlama2Chat70b = 'bedrock/meta.llama2-70b-chat-v1', -} - + BedrockAnthropicClaudeV21 = 'anthropic.claude-v2:1', + BedrockAnthropicClaudeV2 = 'anthropic.claude-v2', + BedrockAnthropicClaudeInstantV1 = 'anthropic.claude-instant-v1', + BedrockAnthropicClaudeV1 = 'anthropic.claude-v1', + BedrockAmazonTitanLite = 'amazon.titan-text-lite-v1', + BedrockAmazonTitanExpress = 'amazon.titan-text-express-v1', + BedrockCohereCommand = 'cohere.command-text-v14', + BedrockAI21J2Mid = 'ai21.j2-mid-v1', + BedrockAI21J2Ultra = 'ai21.j2-ultra-v1', + BedrockMetaLlama2Chat13b = 'meta.llama2-13b-chat-v1', + BedrockMetaLlama2Chat70b = 'meta.llama2-70b-chat-v1', +} + +export type PerplexityAISlug = `perplexity/${PerplexityAIModels}` export enum PerplexityAIModels { - Pplx7bChat = 'perplexity/pplx-7b-chat', - Pplx70bChat = 'perplexity/pplx-70b-chat', - Pplx7bOnline = 'perplexity/pplx-7b-online', - Pplx70bOnline = 'perplexity/pplx-70b-online', - PplxCodeLlama34bInstruct = 'perplexity/codellama-34b-instruct', - PplxLlama213bChat = 'perplexity/llama-2-13b-chat', - PplxLlama270bChat = 'perplexity/llama-2-70b-chat', - PplxMistral7bInstruct = 'perplexity/mistral-7b-instruct', - PplxOpenhermes2Mistral7b = 'perplexity/openhermes-2-mistral-7b', - PplxOpenhermes25Mistral7b = 'perplexity/openhermes-2.5-mistral-7b', - Pplx7bChatAlpha = 'perplexity/pplx-7b-chat-alpha', - Pplx70bChatAlpha = 'perplexity/pplx-70b-chat-alpha', + Pplx7bChat = 'pplx-7b-chat', + Pplx70bChat = 'pplx-70b-chat', + Pplx7bOnline = 'pplx-7b-online', + Pplx70bOnline = 'pplx-70b-online', + PplxCodeLlama34bInstruct = 'codellama-34b-instruct', + PplxLlama213bChat = 'llama-2-13b-chat', + PplxLlama270bChat = 'llama-2-70b-chat', + PplxMistral7bInstruct = 'mistral-7b-instruct', + PplxOpenhermes2Mistral7b = 'openhermes-2-mistral-7b', + PplxOpenhermes25Mistral7b = 'openhermes-2.5-mistral-7b', + Pplx7bChatAlpha = 'pplx-7b-chat-alpha', + Pplx70bChatAlpha = 'pplx-70b-chat-alpha', } export enum VLLMModels { @@ -374,12 +201,15 @@ export enum DeepInfraChatModels { JondurbinAiroborosL270bGpt4141 = 'deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1', } +export type VoyageAISlug = `voyage-ai/${VoyageAIModels | VoyageEmbeddingModels}` export enum VoyageAIModels { - Voyage01 = 'voyage-ai/voyage-01', - VoyageLite01 = 'voyage-ai/voyage-lite-01', - VoyageLite01Instruct = 'voyage-ai/voyage-lite-01-instruct', + Voyage01 = 'voyage-01', + VoyageLite01 = 'voyage-lite-01', + VoyageLite01Instruct = 'voyage-lite-01-instruct', } +export type TogetherAISlug = `together_ai/${TogetherAIModels}` + export enum TogetherAIModels { TogetherLlama270bChat = 'togethercomputer/llama-2-70b-chat', TogetherLlama270b = 'togethercomputer/llama-2-70b', @@ -423,7 +253,36 @@ export enum PetalsModels { HuggyLlamaLlama65 = 'petals/huggyllama/llama-65b', } -export type LLMModels = +export enum AnyscaleModels { + AnyscaleMistral7BInstructV01 = 'anyscale/mistralai/Mistral-7B-Instruct-v0.1', + AnyscaleZephyr7BBeta = 'anyscale/HuggingFaceH4/zephyr-7b-beta', + AnyscaleMetaLlamaLlama27bChatHf = 'anyscale/meta-llama/Llama-2-7b-chat-hf', + AnyscaleMetaLlamaLlama213bChatHf = 'anyscale/meta-llama/Llama-2-13b-chat-hf', + AnyscaleMetaLlamaLlama270bChatHf = 'anyscale/meta-llama/Llama-2-70b-chat-hf', + AnyscaleCodellamaCodeLlama34bInstructHf = 'anyscale/codellama/CodeLlama-34b-Instruct-hf', +} + +export enum OpenRouterModels { + OpenRouterOpenAIGpt35turbo = 'openrouter/openai/gpt-3.5-turbo', + OpenRouterOpenAIGpt35turbo16k = 'openrouter/openai/gpt-3.5-turbo-16k', + OpenRouterOpenAIGpt4 = 'openrouter/openai/gpt-4', + OpenRouterOpenAIGpt432k = 'openrouter/openai/gpt-4-32k', + OpenRouterAnthropicClaud2 = 'openrouter/anthropic/claude-2', + OpenRouterAnthropicClaudInstantV1 = 'openrouter/anthropic/claude-instant-v1', + OpenRouterPalm2ChatBison = 'openrouter/google/palm-2-chat-bison', + OpenRouterPalm2CodeChatBison = 'openrouter/google/palm-2-codechat-bison', + OpenRouterMetaLlamaLlama213bChat = 'openrouter/meta-llama/llama-2-13b-chat', + OpenRouterMetaLlamaLlama270bChat = 'openrouter/meta-llama/llama-2-70b-chat', +} + +export enum ReplicateModels { + ReplicateLlama270bChat = 'replicate/llama-2-70b-chat', + ReplicateLlama213bChat = 'replicate/a16z-infra/llama-2-13b-chat', + ReplicateVicuna13b = 'replicate/vicuna-13b', + ReplicateFlanT5Large = 'replicate/daanelson/flan-t5-large', +} + +export type CompletionModels = | OpenAIChatCompletionModels | OpenAITextCompletionInstructModels | OpenAIVisionModels @@ -432,7 +291,6 @@ export type LLMModels = | OllamaVisionModels | VertexAIGoogleModels | PalmModels - | GoogleAIStudioModels | MistralAIModels | AnthropicModels | SageMakerModels @@ -445,7 +303,10 @@ export type LLMModels = | NLPCloudModels | DeepInfraChatModels | VoyageAIModels - | TogetherAIModels | AlephAlphaModels | BaseTenModels | PetalsModels + | AnyscaleModels + | OpenRouterModels + | GoogleAIStudioModels + | TogetherAIModels diff --git a/plugins/core/src/lib/services/coreLLMService/types/completionTypes.ts b/plugins/core/src/lib/services/coreLLMService/types/completionTypes.ts new file mode 100644 index 0000000000..086dab1f06 --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/types/completionTypes.ts @@ -0,0 +1,26 @@ +import { CompletionModels } from './completionModels' +import { CompletionOptions } from './liteLLMTypes' + +import { Message, Choice } from './messageTypes' + +export type CompletionRequest = { + model: CompletionModels + messages: Message[] + options?: CompletionOptions + api_key?: string +} + +export type CompletionResponse = { + id: string + choices: Choice[] + created: string + model: string + object: string + system_fingerprint: any | null + usage: { + prompt_tokens: number + completion_tokens: number + total_tokens: number + } + _python_object: any +} diff --git a/plugins/core/src/lib/services/coreLLMService/types/liteLLMTypes.ts b/plugins/core/src/lib/services/coreLLMService/types/liteLLMTypes.ts new file mode 100644 index 0000000000..afb1ffacbb --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/types/liteLLMTypes.ts @@ -0,0 +1,42 @@ +export type LiteLLMOptions = { + api_base?: string + api_version?: string + api_key?: string + num_retries?: number + context_window_fallback_dict?: Record + fallbacks?: Array> + metadata?: Record +} + +export type FunctionType = { + name: string + description?: string + parameters: Record +} + +export type ToolType = { + type: string + function: FunctionType +} +export type CompletionOptions = { + temperature?: number + top_p?: number + n?: number + stream?: boolean + stop?: string | string[] + max_tokens?: number + presence_penalty?: number + frequency_penalty?: number + logit_bias?: Record + user?: string + deployment_id?: string + request_timeout?: number + response_format?: { + type: string + } + seed?: number + tools?: ToolType[] + tool_choice?: string | ToolType + functions?: FunctionType[] + function_call?: string +} & LiteLLMOptions diff --git a/plugins/core/src/lib/services/coreLLMService/types/messageTypes.ts b/plugins/core/src/lib/services/coreLLMService/types/messageTypes.ts new file mode 100644 index 0000000000..13d4392f17 --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/types/messageTypes.ts @@ -0,0 +1,35 @@ +import { FunctionType } from './liteLLMTypes' + +export type Message = { + role: string + content: string | null + name?: string + function_call?: { + type: string + function: FunctionType + } +} + +export type Chunk = { + choices: ChunkChoice[] +} + +export type ChunkChoice = { + finish_reason: string + index: number + delta: { + function_call: string + tool_calls: string + content: string + role: string + } +} + +export type Choice = { + finish_reason: string + index: number + message: { + role: string + content: string + } +} diff --git a/plugins/core/src/lib/services/coreLLMService/types/modelProviderMappings.ts b/plugins/core/src/lib/services/coreLLMService/types/modelProviderMappings.ts new file mode 100644 index 0000000000..de354a77ac --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/types/modelProviderMappings.ts @@ -0,0 +1,4 @@ +import { CompletionModels } from './completionModels' +import { LLMProviderKeys } from './providerTypes' + +export type LLMModel = [CompletionModels, LLMProviderKeys] diff --git a/plugins/core/src/lib/services/coreLLMService/types/models.ts b/plugins/core/src/lib/services/coreLLMService/types/models.ts new file mode 100644 index 0000000000..de5b270c59 --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/types/models.ts @@ -0,0 +1,4 @@ +import { EmbeddingModels } from '../../coreEmbeddingService/types' +import { CompletionModels } from './completionModels' + +export type AllModels = CompletionModels | EmbeddingModels diff --git a/plugins/core/src/lib/services/coreLLMService/types/providerTypes.ts b/plugins/core/src/lib/services/coreLLMService/types/providerTypes.ts new file mode 100644 index 0000000000..222a416ebc --- /dev/null +++ b/plugins/core/src/lib/services/coreLLMService/types/providerTypes.ts @@ -0,0 +1,76 @@ +import { PluginCredential } from 'server/credentials' + +export enum LLMProviders { + OpenAI = 'openai', + Azure = 'azure_openai', + Anthropic = 'anthropic', + Sagemaker = 'sagemaker', + Bedrock = 'aws_bedrock', + Anyscale = 'anyscale', + PerplexityAI = 'perplexity', + VLLM = 'vllm', + DeepInfra = 'deepinfra', + Cohere = 'cohere', + TogetherAI = 'together', + AlephAlpha = 'alephalpha', + Baseten = 'baseten', + OpenRouter = 'openrouter', + CustomAPI = 'customapi', + CustomOpenAI = 'custom_openai', + Petals = 'petals', + Ollama = 'ollama', + GoogleAIStudio = 'google', + Palm = 'palm', + HuggingFace = 'Huggingface', + Xinference = 'xinference', + CloudflareWorkersAI = 'cloudflareworkersai', + AI21 = 'ai21', + NLPCloud = 'nlpcloud', + VoyageAI = 'voyageai', + Replicate = 'replicate', + Meta = 'meta', + Mistral = 'mistralai', + VertexAI = 'vertexai', +} + +export enum LLMProviderKeys { + OpenAI = 'OPENAI_API_KEY', + Azure = 'AZURE_API_KEY', + Anthropic = 'ANTHROPIC_API_KEY', + Sagemaker = 'SAGEMAKER_API_KEY', + Bedrock = 'BEDROCK_API_KEY', + Anyscale = 'ANYSCALE_API_KEY', + PerplexityAI = 'PERPLEXITY_API_KEY', + VLLM = 'VLLM_API_KEY', + DeepInfra = 'DEEPINFRA_API_KEY', + Cohere = 'COHERE_API_KEY', + TogetherAI = 'TOGETHER_AI_API_KEY', + AlephAlpha = 'ALEPH_ALPHA_API_KEY', + Baseten = 'BASETEN_API_KEY', + OpenRouter = 'OPENROUTER_API_KEY', + CustomAPI = 'CUSTOM_API_KEY', + CustomOpenAI = 'CUSTOM_OPEN_API_KEY', + Petals = 'PETALS_API_KEY', + Ollama = 'OLLAMA_API_KEY', + GoogleAIStudio = 'GOOGLEAISTUDIO_API_KEY', + Palm = 'PALM_API_KEY', + HuggingFace = 'HUGGINGFACE_API_KEY', + Xinference = 'XINFERENCE_API_KEY', + CloudflareWorkersAI = 'CLOUDFLAREWORKERSAI_API_KEY', + AI21 = 'AI21_API_KEY', + NLPCloud = 'NLPCLOUD_API_KEY', + VoyageAI = 'VOYAGEAI_API_KEY', + Replicate = 'REPLICATE_API_KEY', + Meta = 'META_API_KEY', + Mistral = 'MISTRAL_API_KEY', + VertexAI = 'VERTEXAI_API_KEY', + Unknown = 'unknown', +} + +export interface ModelProviderMapping { + provider: LLMProviders + apiKeyName: LLMProviderKeys +} +export type LLMCredential = PluginCredential & { + value: string +} diff --git a/plugins/core/src/lib/services/coreMemoryService/coreMemoryService.ts b/plugins/core/src/lib/services/coreMemoryService/coreMemoryService.ts index 33851bbd02..9a55a0e830 100644 --- a/plugins/core/src/lib/services/coreMemoryService/coreMemoryService.ts +++ b/plugins/core/src/lib/services/coreMemoryService/coreMemoryService.ts @@ -1,19 +1,22 @@ import { python } from 'pythonia' import { PRODUCTION } from 'shared/config' -import { - LLMCredential, - LLMModels, - LLMProviderKeys, - Models, - OpenAIChatCompletionModels, -} from '../coreLLMService/types' -import { modelProviderMap } from '../coreLLMService/constants' + import { DataType } from './coreMemoryTypes' import { EmbeddingModels, OpenAIEmbeddingModels, } from '../coreEmbeddingService/types' -import { embeddingProviderMap } from '../coreEmbeddingService/constants' +import { + findProviderKey, + findProviderName, +} from '../coreLLMService/findProvider' +import { + OpenAIChatCompletionModels, + CompletionModels, +} from '../coreLLMService/types/completionModels' + +import { LLMCredential } from '../coreLLMService/types/providerTypes' +import { AllModels } from '../coreLLMService/types/models' export interface ICoreMemoryService { initialize(agentId: string): Promise @@ -76,6 +79,11 @@ class CoreMemoryService { // Use Pythonia to create an instance of the Embedchain App this.embedchain = await python('embedchain') + console.log( + '#########Pythonia Embedchain:', + OpenAIChatCompletionModels.GPT35Turbo, + OpenAIEmbeddingModels.TextEmbeddingAda002 + ) // Ste initial LLM and Embedder models this.setLLM(OpenAIChatCompletionModels.GPT35Turbo) this.setEmbedder(OpenAIEmbeddingModels.TextEmbeddingAda002) @@ -93,12 +101,12 @@ class CoreMemoryService { } } - setModel(model: LLMModels) { + setModel(model: CompletionModels) { this.setLLM(model) } - private setLLM(model: LLMModels) { - const providerName = this.findProviderName(model) + private setLLM(model: CompletionModels) { + const providerName = findProviderName(model) const credential = this.getCredential(model) const params = this.changeLLMParams() @@ -113,7 +121,7 @@ class CoreMemoryService { } private setEmbedder(model: EmbeddingModels) { - const providerName = this.findEmbeddingProviderName(model) + const providerName = findProviderName(model) const credential = this.getCredential(model) this.baseConfig.embedder = { @@ -136,18 +144,6 @@ class CoreMemoryService { return newParams } - private findProvider = (model: Models): LLMProviderKeys => { - return modelProviderMap[model] - } - - private findProviderName = (model: Models) => { - return modelProviderMap[model] - } - - private findEmbeddingProviderName = (model: EmbeddingModels) => { - return embeddingProviderMap[model] - } - addCredential(credential: LLMCredential): void { const existingCredentialIndex = this.credentials.findIndex( c => c.serviceType === credential.serviceType @@ -164,13 +160,13 @@ class CoreMemoryService { return Object.values(DataType) } - private getCredential(model: Models): string { - const provider = this.findProvider(model) + private getCredential(model: AllModels): string { + const provider = findProviderKey(model) let credential = this.credentials.find( c => c.serviceType === provider )?.value - if (!credential && !PRODUCTION) { + if (!credential && !PRODUCTION && provider) { credential = process.env[provider] } diff --git a/plugins/core/src/lib/services/types.ts b/plugins/core/src/lib/services/types.ts index 95c9d31a3c..57c15ea070 100644 --- a/plugins/core/src/lib/services/types.ts +++ b/plugins/core/src/lib/services/types.ts @@ -1,10 +1,9 @@ +import { CompletionModels } from './coreLLMService/types/completionModels' import { - Chunk, - CompletionRequest, CompletionResponse, - LLMModels, - Message, -} from './coreLLMService/types' + CompletionRequest, +} from './coreLLMService/types/completionTypes' +import { Chunk, Message } from './coreLLMService/types/messageTypes' export interface IBudgetManagerService { // Creates a budget for a user @@ -23,7 +22,7 @@ export interface IBudgetManagerService { // Returns the current cost of a user get_current_cost(user: string): Promise // Returns the model cost of a user - get_model_cost(user: string): Promise> + get_model_cost(user: string): Promise> // Checks if a user is valid is_valid_user(user: string): Promise // Returns a list of all users @@ -69,7 +68,7 @@ export interface ICoreBudgetManagerService { completionObj: CompletionResponse ): Promise getCurrentCost(projectId: string): Promise - getModelCost(projectId: string): Promise> + getModelCost(projectId: string): Promise> isValidUser(projectId: string): Promise getUsers(): Promise resetCost(projectId: string): Promise diff --git a/poetry.lock b/poetry.lock index d095662264..81c885b70a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -573,43 +573,43 @@ files = [ [[package]] name = "cryptography" -version = "42.0.0" +version = "42.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:c640b0ef54138fde761ec99a6c7dc4ce05e80420262c20fa239e694ca371d434"}, - {file = "cryptography-42.0.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:678cfa0d1e72ef41d48993a7be75a76b0725d29b820ff3cfd606a5b2b33fda01"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:146e971e92a6dd042214b537a726c9750496128453146ab0ee8971a0299dc9bd"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87086eae86a700307b544625e3ba11cc600c3c0ef8ab97b0fda0705d6db3d4e3"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0a68bfcf57a6887818307600c3c0ebc3f62fbb6ccad2240aa21887cda1f8df1b"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5a217bca51f3b91971400890905a9323ad805838ca3fa1e202a01844f485ee87"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ca20550bb590db16223eb9ccc5852335b48b8f597e2f6f0878bbfd9e7314eb17"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:33588310b5c886dfb87dba5f013b8d27df7ffd31dc753775342a1e5ab139e59d"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9515ea7f596c8092fdc9902627e51b23a75daa2c7815ed5aa8cf4f07469212ec"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:35cf6ed4c38f054478a9df14f03c1169bb14bd98f0b1705751079b25e1cb58bc"}, - {file = "cryptography-42.0.0-cp37-abi3-win32.whl", hash = "sha256:8814722cffcfd1fbd91edd9f3451b88a8f26a5fd41b28c1c9193949d1c689dc4"}, - {file = "cryptography-42.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:a2a8d873667e4fd2f34aedab02ba500b824692c6542e017075a2efc38f60a4c0"}, - {file = "cryptography-42.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:8fedec73d590fd30c4e3f0d0f4bc961aeca8390c72f3eaa1a0874d180e868ddf"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be41b0c7366e5549265adf2145135dca107718fa44b6e418dc7499cfff6b4689"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ca482ea80626048975360c8e62be3ceb0f11803180b73163acd24bf014133a0"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c58115384bdcfe9c7f644c72f10f6f42bed7cf59f7b52fe1bf7ae0a622b3a139"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:56ce0c106d5c3fec1038c3cca3d55ac320a5be1b44bf15116732d0bc716979a2"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:324721d93b998cb7367f1e6897370644751e5580ff9b370c0a50dc60a2003513"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:d97aae66b7de41cdf5b12087b5509e4e9805ed6f562406dfcf60e8481a9a28f8"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:85f759ed59ffd1d0baad296e72780aa62ff8a71f94dc1ab340386a1207d0ea81"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:206aaf42e031b93f86ad60f9f5d9da1b09164f25488238ac1dc488334eb5e221"}, - {file = "cryptography-42.0.0-cp39-abi3-win32.whl", hash = "sha256:74f18a4c8ca04134d2052a140322002fef535c99cdbc2a6afc18a8024d5c9d5b"}, - {file = "cryptography-42.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:14e4b909373bc5bf1095311fa0f7fcabf2d1a160ca13f1e9e467be1ac4cbdf94"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3005166a39b70c8b94455fdbe78d87a444da31ff70de3331cdec2c568cf25b7e"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:be14b31eb3a293fc6e6aa2807c8a3224c71426f7c4e3639ccf1a2f3ffd6df8c3"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:bd7cf7a8d9f34cc67220f1195884151426ce616fdc8285df9054bfa10135925f"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c310767268d88803b653fffe6d6f2f17bb9d49ffceb8d70aed50ad45ea49ab08"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bdce70e562c69bb089523e75ef1d9625b7417c6297a76ac27b1b8b1eb51b7d0f"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e9326ca78111e4c645f7e49cbce4ed2f3f85e17b61a563328c85a5208cf34440"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:69fd009a325cad6fbfd5b04c711a4da563c6c4854fc4c9544bff3088387c77c0"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:988b738f56c665366b1e4bfd9045c3efae89ee366ca3839cd5af53eaa1401bce"}, - {file = "cryptography-42.0.0.tar.gz", hash = "sha256:6cf9b76d6e93c62114bd19485e5cb003115c134cf9ce91f8ac924c44f8c8c3f4"}, + {file = "cryptography-42.0.1-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:265bdc693570b895eb641410b8fc9e8ddbce723a669236162b9d9cfb70bd8d77"}, + {file = "cryptography-42.0.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:160fa08dfa6dca9cb8ad9bd84e080c0db6414ba5ad9a7470bc60fb154f60111e"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:727387886c9c8de927c360a396c5edcb9340d9e960cda145fca75bdafdabd24c"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d84673c012aa698555d4710dcfe5f8a0ad76ea9dde8ef803128cc669640a2e0"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e6edc3a568667daf7d349d7e820783426ee4f1c0feab86c29bd1d6fe2755e009"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:d50718dd574a49d3ef3f7ef7ece66ef281b527951eb2267ce570425459f6a404"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9544492e8024f29919eac2117edd8c950165e74eb551a22c53f6fdf6ba5f4cb8"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ab6b302d51fbb1dd339abc6f139a480de14d49d50f65fdc7dff782aa8631d035"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2fe16624637d6e3e765530bc55caa786ff2cbca67371d306e5d0a72e7c3d0407"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ed1b2130f5456a09a134cc505a17fc2830a1a48ed53efd37dcc904a23d7b82fa"}, + {file = "cryptography-42.0.1-cp37-abi3-win32.whl", hash = "sha256:e5edf189431b4d51f5c6fb4a95084a75cef6b4646c934eb6e32304fc720e1453"}, + {file = "cryptography-42.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:6bfd823b336fdcd8e06285ae8883d3d2624d3bdef312a0e2ef905f332f8e9302"}, + {file = "cryptography-42.0.1-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:351db02c1938c8e6b1fee8a78d6b15c5ccceca7a36b5ce48390479143da3b411"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:430100abed6d3652208ae1dd410c8396213baee2e01a003a4449357db7dc9e14"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dff7a32880a51321f5de7869ac9dde6b1fca00fc1fef89d60e93f215468e824"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b512f33c6ab195852595187af5440d01bb5f8dd57cb7a91e1e009a17f1b7ebca"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:95d900d19a370ae36087cc728e6e7be9c964ffd8cbcb517fd1efb9c9284a6abc"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:6ac8924085ed8287545cba89dc472fc224c10cc634cdf2c3e2866fe868108e77"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cb2861a9364fa27d24832c718150fdbf9ce6781d7dc246a516435f57cfa31fe7"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25ec6e9e81de5d39f111a4114193dbd39167cc4bbd31c30471cebedc2a92c323"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9d61fcdf37647765086030d81872488e4cb3fafe1d2dda1d487875c3709c0a49"}, + {file = "cryptography-42.0.1-cp39-abi3-win32.whl", hash = "sha256:16b9260d04a0bfc8952b00335ff54f471309d3eb9d7e8dbfe9b0bd9e26e67881"}, + {file = "cryptography-42.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:7911586fc69d06cd0ab3f874a169433db1bc2f0e40988661408ac06c4527a986"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d3594947d2507d4ef7a180a7f49a6db41f75fb874c2fd0e94f36b89bfd678bf2"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8d7efb6bf427d2add2f40b6e1e8e476c17508fa8907234775214b153e69c2e11"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:126e0ba3cc754b200a2fb88f67d66de0d9b9e94070c5bc548318c8dab6383cb6"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:802d6f83233cf9696b59b09eb067e6b4d5ae40942feeb8e13b213c8fad47f1aa"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0b7cacc142260ada944de070ce810c3e2a438963ee3deb45aa26fd2cee94c9a4"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:32ea63ceeae870f1a62e87f9727359174089f7b4b01e4999750827bf10e15d60"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d3902c779a92151f134f68e555dd0b17c658e13429f270d8a847399b99235a3f"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:50aecd93676bcca78379604ed664c45da82bc1241ffb6f97f6b7392ed5bc6f04"}, + {file = "cryptography-42.0.1.tar.gz", hash = "sha256:fd33f53809bb363cf126bebe7a99d97735988d9b0131a2be59fbf83e1259a5b7"}, ] [package.dependencies] @@ -670,13 +670,13 @@ files = [ [[package]] name = "embedchain" -version = "0.1.68" +version = "0.1.69" description = "Simplest open source retrieval(RAG) framework" optional = false python-versions = ">=3.9,<3.12" files = [ - {file = "embedchain-0.1.68-py3-none-any.whl", hash = "sha256:ec5ab00dcf197295b6fc0fa2f2755068749189246a5caba194828f27a29be310"}, - {file = "embedchain-0.1.68.tar.gz", hash = "sha256:f36a2e0214d6a054452a8fdaa427aa6bf31feff4587951aa05c30b5b7465ab1f"}, + {file = "embedchain-0.1.69-py3-none-any.whl", hash = "sha256:b77cb460eb75b163e3ed46a48e10b0d457b44da7b90d9232b9e4db9ddbd92eea"}, + {file = "embedchain-0.1.69.tar.gz", hash = "sha256:77a3ec9e20b276f92a1cf6dfb0f4cd59208e08894bdd83b4b569f4e271ceb2f6"}, ] [package.dependencies] @@ -894,13 +894,13 @@ tqdm = ["tqdm"] [[package]] name = "google-auth" -version = "2.26.2" +version = "2.27.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google-auth-2.26.2.tar.gz", hash = "sha256:97327dbbf58cccb58fc5a1712bba403ae76668e64814eb30f7316f7e27126b81"}, - {file = "google_auth-2.26.2-py2.py3-none-any.whl", hash = "sha256:3f445c8ce9b61ed6459aad86d8ccdba4a9afed841b2d1451a11ef4db08957424"}, + {file = "google-auth-2.27.0.tar.gz", hash = "sha256:e863a56ccc2d8efa83df7a80272601e43487fa9a728a376205c86c26aaefa821"}, + {file = "google_auth-2.27.0-py2.py3-none-any.whl", hash = "sha256:8e4bad367015430ff253fe49d500fdc3396c1a434db5740828c728e45bcce245"}, ] [package.dependencies] @@ -1778,13 +1778,13 @@ sympy = "*" [[package]] name = "openai" -version = "1.9.0" +version = "1.10.0" description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.9.0-py3-none-any.whl", hash = "sha256:5774a0582ed82f6de92200ed5024e03e272b93e04e9d31caeda5fb80f63df50d"}, - {file = "openai-1.9.0.tar.gz", hash = "sha256:3e9947a544556c051fa138a4def5bd8b468364ec52803c6628532ab949ddce55"}, + {file = "openai-1.10.0-py3-none-any.whl", hash = "sha256:aa69e97d0223ace9835fbf9c997abe9ee95318f684fd2de6d02c870700c71ebc"}, + {file = "openai-1.10.0.tar.gz", hash = "sha256:208886cb501b930dc63f48d51db9c15e5380380f80516d07332adad67c9f1053"}, ] [package.dependencies] @@ -1902,13 +1902,13 @@ files = [ [[package]] name = "posthog" -version = "3.3.2" +version = "3.3.3" description = "Integrate PostHog into any python application." optional = false python-versions = "*" files = [ - {file = "posthog-3.3.2-py2.py3-none-any.whl", hash = "sha256:14fb43ea95c40b353db59c49af2c09ff15188aa2963f48091fc7912fa9375263"}, - {file = "posthog-3.3.2.tar.gz", hash = "sha256:734bf89f3c372605a8bbf2b07f600885287209145d747b09ccd004c59834750e"}, + {file = "posthog-3.3.3-py2.py3-none-any.whl", hash = "sha256:22b450be8cfdbcf403cbbcc50e9ca6e681d04c6a057336483cc4b8beb5794a15"}, + {file = "posthog-3.3.3.tar.gz", hash = "sha256:2f16372675a81fb093dd19d7fb8ea8166068f5a7a17358d71ccb2f7080a4c8c1"}, ] [package.dependencies]