From ad5e17cd7c8d1e6cb3d8359a502a1eac97cc0859 Mon Sep 17 00:00:00 2001 From: Kasper Marstal Date: Thu, 23 Jan 2025 10:54:02 +0100 Subject: [PATCH] bug: Fix OpenAiCompatible's use and configuration of base address and api key (#92) --- README.md | 6 +++++- src/Cellm/Models/Client.cs | 4 ++-- .../OpenAiCompatibleRequest.cs | 2 +- .../OpenAiCompatibleRequestHandler.cs | 10 +++++++--- .../appsettings.Local.OpenAiCompatible.json | 10 ++++++++++ src/Cellm/appsettings.json | 19 ++++++++++++++----- 6 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 src/Cellm/appsettings.Local.OpenAiCompatible.json diff --git a/README.md b/README.md index abc2f04..d4add9e 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,10 @@ These use cases are starting points. Experiment with different instructions to f ## Models +Cellm supports both hosted and local models. These are configured via appsettings files. In general, you should leave `appsettings.json` alone and add your own configuration to `appsettings.Local.json` only. Any settings in this file will override the default settings in `appsettings.json`. + +You can use `appsettings.Local.OpenAiCompatible.json` as a starting point for configuring any model provider that is compatible with OpenAI's API. Just rename it to `appsettings.Local.json` and edit the values. The following sections shows you how to configure `appsettings.Local.json` for commonly used hosted and local models. + ### Hosted LLMs Cellm supports hosted models from Anthropic, Google, OpenAI, and any OpenAI-compatible provider. To use e.g. Claude 3.5 Sonnet from Anthropic: @@ -221,7 +225,7 @@ Llamafile is a stand-alone executable that is very easy to setup. To get started 3. Rename `appsettings.Llamafile.json` to `appsettings.Local.json`. 4. Build and install Cellm. -### Dockerized Ollama and vLLM +#### Dockerized Ollama and vLLM Both Ollama and vLLM are packaged up with docker compose files in the `docker/` folder. Ollama is designed for easy of use and vLLM is designed to run many requests in parallel which is particularly useful for this Add-In. Open WebUI in included in both Ollama and vLLM docker compose files so you can test the local model outside of Cellm. It is available at `http://localhost:3000`. diff --git a/src/Cellm/Models/Client.cs b/src/Cellm/Models/Client.cs index c7ed174..bd75ae8 100644 --- a/src/Cellm/Models/Client.cs +++ b/src/Cellm/Models/Client.cs @@ -11,7 +11,7 @@ namespace Cellm.Models; -public class Client(ISender sender) +internal class Client(ISender sender) { public async Task Send(Prompt prompt, Provider? provider, Uri? baseAddress, CancellationToken cancellationToken) { @@ -23,7 +23,7 @@ public async Task Send(Prompt prompt, Provider? provider, Uri? baseAddre Provider.Llamafile => await sender.Send(new LlamafileRequest(prompt), cancellationToken), Provider.Ollama => await sender.Send(new OllamaRequest(prompt), cancellationToken), Provider.OpenAi => await sender.Send(new OpenAiRequest(prompt), cancellationToken), - Provider.OpenAiCompatible => await sender.Send(new OpenAiCompatibleRequest(prompt, baseAddress ?? throw new NullReferenceException($"{nameof(Provider.OpenAiCompatible)} requires BaseAddress")), cancellationToken), + Provider.OpenAiCompatible => await sender.Send(new OpenAiCompatibleRequest(prompt, baseAddress), cancellationToken), _ => throw new NotSupportedException($"Provider {provider} is not supported") }; diff --git a/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequest.cs b/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequest.cs index 1c12b2d..3502670 100644 --- a/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequest.cs +++ b/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequest.cs @@ -4,5 +4,5 @@ namespace Cellm.Models.Providers.OpenAiCompatible; internal record OpenAiCompatibleRequest( Prompt Prompt, - Uri BaseAddress, + Uri? BaseAddress, string? ApiKey = null) : IModelRequest; diff --git a/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequestHandler.cs b/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequestHandler.cs index 3a51b36..7115673 100644 --- a/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequestHandler.cs +++ b/src/Cellm/Models/Providers/OpenAiCompatible/OpenAiCompatibleRequestHandler.cs @@ -1,18 +1,22 @@ using Cellm.Models.Prompts; +using Microsoft.Extensions.Options; namespace Cellm.Models.Providers.OpenAiCompatible; internal class OpenAiCompatibleRequestHandler( - OpenAiCompatibleChatClientFactory openAiCompatibleChatClientFactory) + OpenAiCompatibleChatClientFactory openAiCompatibleChatClientFactory, + IOptions openAiCompatibleConfiguration) : IModelRequestHandler { public async Task Handle(OpenAiCompatibleRequest request, CancellationToken cancellationToken) { + var api_key = string.IsNullOrEmpty(request.ApiKey) ? "API_KEY" : request.ApiKey; + var chatClient = openAiCompatibleChatClientFactory.Create( - request.BaseAddress, + request.BaseAddress ?? openAiCompatibleConfiguration.Value.BaseAddress, request.Prompt.Options.ModelId ?? string.Empty, - request.ApiKey ?? "API_KEY"); + api_key); var chatCompletion = await chatClient.CompleteAsync(request.Prompt.Messages, request.Prompt.Options, cancellationToken); diff --git a/src/Cellm/appsettings.Local.OpenAiCompatible.json b/src/Cellm/appsettings.Local.OpenAiCompatible.json new file mode 100644 index 0000000..cf301da --- /dev/null +++ b/src/Cellm/appsettings.Local.OpenAiCompatible.json @@ -0,0 +1,10 @@ +{ + "OpenAiCompatibleConfiguration": { + "BaseAddress": "YOUR_API_ENDPOINT", + "DefaultModel": "YOUR_DEFAULT_MODEL", + "ApiKey": "YOUR_API_KEY_OPTIONAL" + }, + "ProviderConfiguration": { + "DefaultProvider": "OpenAiCompatible" + } +} diff --git a/src/Cellm/appsettings.json b/src/Cellm/appsettings.json index 58b0e98..2c971a6 100644 --- a/src/Cellm/appsettings.json +++ b/src/Cellm/appsettings.json @@ -2,19 +2,28 @@ "AnthropicConfiguration": { "BaseAddress": "https://api.anthropic.com", "DefaultModel": "claude-3-5-sonnet-latest", - "Version": "2023-06-01" + "Version": "2023-06-01", + "ApiKey": "YOUR_ANTHROPIC_API_KEY" }, "LlamafileConfiguration": { - "BaseAddress": "http://127.0.0.1/v1", - "DefaultModel": "gemma-2-2b" + "BaseAddress": "http://127.0.0.1:8080/v1", + "DefaultModel": "gemma-2-2b", + "ApiKey": "YOUR_LLAMAFILE_API_KEY_OPTIONAL" }, "OllamaConfiguration": { "BaseAddress": "http://127.0.0.1:11434", - "DefaultModel": "gemma2:2b" + "DefaultModel": "gemma2:2b", + "ApiKey": "YOUR_OLLAMA_API_KEY_OPTIONAL" }, "OpenAiConfiguration": { "BaseAddress": "https://api.openai.com/v1", - "DefaultModel": "gpt-4o-mini" + "DefaultModel": "gpt-4o-mini", + "ApiKey": "YOUR_OPENAI_API_KEY" + }, + "OpenAiCompatibleConfiguration": { + "BaseAddress": "https://api.openai.com/v1", + "DefaultModel": "gpt-4o-mini", + "ApiKey": "YOUR_API_KEY" }, "CellmConfiguration": { "Debug": false