o1 supports system prompt

sdcb · Dec 27, 2024 · df75c64 · df75c64
1 parent 76f1bb4
commit df75c64
Show file tree

Hide file tree

Showing 4 changed files with 52 additions and 5 deletions.
diff --git a/src/BE/DB/Extensions/ModelReference.cs b/src/BE/DB/Extensions/ModelReference.cs
@@ -7,4 +7,6 @@ public partial class ModelReference
         if (temperature == null) return null;
         return temperature * (float)(MaxTemperature - MinTemperature) + (float)MinTemperature;
     }
+
+    public bool IsSdkUnsupportedO1 => Name == "o1-2024-12-17";
 }
diff --git a/src/BE/DB/Init/BasicData.cs b/src/BE/DB/Init/BasicData.cs
@@ -107,7 +107,7 @@ private static void InsertTransactionTypes(ChatsDB db)
 
     private static void InsertModelReferences(ChatsDB db)
     {
-        // Generated from data, hash: c8beb20f27ffd4e4e2a71aa2e3af7741707f29f25d840373fd3ee5d37a3d9238
+        // Generated from data, hash: 0c2bed38deb285776af261f503e3dd7a41305b0030d209492e606f550b472c33
         db.ModelReferences.AddRange(
         [
             new(){ Id=0,    ProviderId=0,  Name="Test",                          ShortName=null,                   IsLegacy=false, MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=2048,     MaxResponseTokens=2048,   TokenizerId=1,    InputTokenPrice1M=0.00000M,   OutputTokenPrice1M=0.00000M,   CurrencyCode="RMB", },
@@ -129,7 +129,7 @@ private static void InsertModelReferences(ChatsDB db)
             new(){ Id=115,  ProviderId=1,  Name="gpt-4o-2024-05-13",             ShortName="gpt-4o",               IsLegacy=true,  MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=128000,   MaxResponseTokens=4096,   TokenizerId=2,    InputTokenPrice1M=5.00000M,   OutputTokenPrice1M=15.00000M,  CurrencyCode="USD", },
             new(){ Id=116,  ProviderId=1,  Name="gpt-4o-2024-08-06",             ShortName="gpt-4o",               IsLegacy=false, MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=128000,   MaxResponseTokens=16384,  TokenizerId=2,    InputTokenPrice1M=2.50000M,   OutputTokenPrice1M=10.00000M,  CurrencyCode="USD", },
             new(){ Id=117,  ProviderId=1,  Name="gpt-4o-2024-11-20",             ShortName="gpt-4o",               IsLegacy=false, MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=128000,   MaxResponseTokens=16384,  TokenizerId=2,    InputTokenPrice1M=2.50000M,   OutputTokenPrice1M=10.00000M,  CurrencyCode="USD", },
-            new(){ Id=118,  ProviderId=1,  Name="o1-2024-12-17",                 ShortName="o1",                   IsLegacy=false, MinTemperature=1.00M, MaxTemperature=1.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=false, AllowStreaming=false, ContextWindow=200000,   MaxResponseTokens=100000, TokenizerId=2,    InputTokenPrice1M=15.00000M,  OutputTokenPrice1M=60.00000M,  CurrencyCode="USD", },
+            new(){ Id=118,  ProviderId=1,  Name="o1-2024-12-17",                 ShortName="o1",                   IsLegacy=false, MinTemperature=1.00M, MaxTemperature=1.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=false, ContextWindow=200000,   MaxResponseTokens=100000, TokenizerId=2,    InputTokenPrice1M=15.00000M,  OutputTokenPrice1M=60.00000M,  CurrencyCode="USD", },
             new(){ Id=200,  ProviderId=2,  Name="hunyuan-turbo",                 ShortName=null,                   IsLegacy=false, MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=true,  AllowVision=false, AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=32768,    MaxResponseTokens=4096,   TokenizerId=null, InputTokenPrice1M=15.00000M,  OutputTokenPrice1M=50.00000M,  CurrencyCode="RMB", },
             new(){ Id=201,  ProviderId=2,  Name="hunyuan-pro",                   ShortName=null,                   IsLegacy=false, MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=true,  AllowVision=false, AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=32768,    MaxResponseTokens=4096,   TokenizerId=null, InputTokenPrice1M=30.00000M,  OutputTokenPrice1M=100.00000M, CurrencyCode="RMB", },
             new(){ Id=202,  ProviderId=2,  Name="hunyuan-standard-256K",         ShortName=null,                   IsLegacy=false, MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=true,  AllowVision=false, AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=262144,   MaxResponseTokens=6144,   TokenizerId=null, InputTokenPrice1M=15.00000M,  OutputTokenPrice1M=60.00000M,  CurrencyCode="RMB", },
@@ -169,7 +169,7 @@ private static void InsertModelReferences(ChatsDB db)
             new(){ Id=515,  ProviderId=5,  Name="gpt-4o-2024-05-13",             ShortName="gpt-4o",               IsLegacy=true,  MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=128000,   MaxResponseTokens=4096,   TokenizerId=2,    InputTokenPrice1M=5.00000M,   OutputTokenPrice1M=15.00000M,  CurrencyCode="USD", },
             new(){ Id=516,  ProviderId=5,  Name="gpt-4o-2024-08-06",             ShortName="gpt-4o",               IsLegacy=false, MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=128000,   MaxResponseTokens=16384,  TokenizerId=2,    InputTokenPrice1M=2.50000M,   OutputTokenPrice1M=10.00000M,  CurrencyCode="USD", },
             new(){ Id=517,  ProviderId=5,  Name="gpt-4o-2024-11-20",             ShortName="gpt-4o",               IsLegacy=false, MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=128000,   MaxResponseTokens=16384,  TokenizerId=2,    InputTokenPrice1M=2.50000M,   OutputTokenPrice1M=10.00000M,  CurrencyCode="USD", },
-            new(){ Id=518,  ProviderId=5,  Name="o1-2024-12-17",                 ShortName="o1",                   IsLegacy=false, MinTemperature=1.00M, MaxTemperature=1.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=false, AllowStreaming=false, ContextWindow=200000,   MaxResponseTokens=100000, TokenizerId=2,    InputTokenPrice1M=15.00000M,  OutputTokenPrice1M=60.00000M,  CurrencyCode="USD", },
+            new(){ Id=518,  ProviderId=5,  Name="o1-2024-12-17",                 ShortName="o1",                   IsLegacy=false, MinTemperature=1.00M, MaxTemperature=1.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=false, ContextWindow=200000,   MaxResponseTokens=100000, TokenizerId=2,    InputTokenPrice1M=15.00000M,  OutputTokenPrice1M=60.00000M,  CurrencyCode="USD", },
             new(){ Id=600,  ProviderId=6,  Name="ernie-4.0-turbo-8k",            ShortName="ERNIE-4.0-Turbo-8K",   IsLegacy=false, MinTemperature=0.00M, MaxTemperature=1.00M, AllowSearch=true,  AllowVision=false, AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=8192,     MaxResponseTokens=2048,   TokenizerId=null, InputTokenPrice1M=30.00000M,  OutputTokenPrice1M=90.00000M,  CurrencyCode="RMB", },
             new(){ Id=601,  ProviderId=6,  Name="completions_pro",               ShortName="ERNIE-4.0-8K",         IsLegacy=false, MinTemperature=0.00M, MaxTemperature=1.00M, AllowSearch=true,  AllowVision=false, AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=8192,     MaxResponseTokens=2048,   TokenizerId=null, InputTokenPrice1M=20.00000M,  OutputTokenPrice1M=60.00000M,  CurrencyCode="RMB", },
             new(){ Id=602,  ProviderId=6,  Name="completions",                   ShortName="ERNIE-3.5-8K",         IsLegacy=false, MinTemperature=0.00M, MaxTemperature=1.00M, AllowSearch=true,  AllowVision=false, AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=8192,     MaxResponseTokens=2048,   TokenizerId=null, InputTokenPrice1M=0.80000M,   OutputTokenPrice1M=2.00000M,   CurrencyCode="RMB", },
@@ -244,7 +244,7 @@ private static void InsertModelReferences(ChatsDB db)
             new(){ Id=1217, ProviderId=12, Name="Phi-3.5-vision-instruct",       ShortName="Phi-3.5",              IsLegacy=false, MinTemperature=0.00M, MaxTemperature=2.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=true,  ContextWindow=131072,   MaxResponseTokens=4096,   TokenizerId=null, InputTokenPrice1M=1.00000M,   OutputTokenPrice1M=2.00000M,   CurrencyCode="USD", },
             new(){ Id=1218, ProviderId=12, Name="o1-preview",                    ShortName=null,                   IsLegacy=true,  MinTemperature=1.00M, MaxTemperature=1.00M, AllowSearch=false, AllowVision=false, AllowSystemPrompt=false, AllowStreaming=false, ContextWindow=128000,   MaxResponseTokens=32768,  TokenizerId=null, InputTokenPrice1M=15.00000M,  OutputTokenPrice1M=60.00000M,  CurrencyCode="USD", },
             new(){ Id=1219, ProviderId=12, Name="o1-mini",                       ShortName=null,                   IsLegacy=false, MinTemperature=1.00M, MaxTemperature=1.00M, AllowSearch=false, AllowVision=false, AllowSystemPrompt=false, AllowStreaming=false, ContextWindow=128000,   MaxResponseTokens=65536,  TokenizerId=null, InputTokenPrice1M=3.00000M,   OutputTokenPrice1M=12.00000M,  CurrencyCode="USD", },
-            new(){ Id=1220, ProviderId=12, Name="o1-2024-12-17",                 ShortName="o1",                   IsLegacy=false, MinTemperature=1.00M, MaxTemperature=1.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=false, AllowStreaming=false, ContextWindow=200000,   MaxResponseTokens=100000, TokenizerId=2,    InputTokenPrice1M=15.00000M,  OutputTokenPrice1M=60.00000M,  CurrencyCode="USD", }
+            new(){ Id=1220, ProviderId=12, Name="o1-2024-12-17",                 ShortName="o1",                   IsLegacy=false, MinTemperature=1.00M, MaxTemperature=1.00M, AllowSearch=false, AllowVision=true,  AllowSystemPrompt=true,  AllowStreaming=false, ContextWindow=200000,   MaxResponseTokens=100000, TokenizerId=2,    InputTokenPrice1M=15.00000M,  OutputTokenPrice1M=60.00000M,  CurrencyCode="USD", }
         ]);
     }
 

diff --git a/src/BE/Services/ChatServices/Implementations/OpenAI/AzureChatService.cs b/src/BE/Services/ChatServices/Implementations/OpenAI/AzureChatService.cs
@@ -15,7 +15,7 @@ static ChatClient CreateChatClient(Model model)
         ArgumentException.ThrowIfNullOrWhiteSpace(model.ModelKey.Secret, nameof(model.ModelKey.Secret));
 
         AzureOpenAIClientOptions options = new();
-        if (model.ModelReference.Name == "o1-2024-12-17")
+        if (model.ModelReference.IsSdkUnsupportedO1)
         {
             // o1 only supports api version: 2024-12-01-preview
             options

diff --git a/src/BE/Services/ChatServices/Implementations/OpenAI/OpenAIChatService.cs b/src/BE/Services/ChatServices/Implementations/OpenAI/OpenAIChatService.cs
@@ -4,6 +4,8 @@
 using System.Runtime.CompilerServices;
 using System.ClientModel;
 using Chats.BE.DB;
+using System.ClientModel.Primitives;
+using System.Text.Json;
 
 namespace Chats.BE.Services.ChatServices.Implementations.OpenAI;
 
@@ -49,6 +51,16 @@ public override async IAsyncEnumerable<ChatSegment> ChatStreamed(IReadOnlyList<C
 
     public override async Task<ChatSegment> Chat(IReadOnlyList<ChatMessage> messages, ChatCompletionOptions options, CancellationToken cancellationToken)
     {
+        if (Model.ModelReference.IsSdkUnsupportedO1)
+        {
+            // must use replace system chat message into developer chat message for unsupported model
+            messages = messages.Select(m => m switch
+            {
+                SystemChatMessage sys => new DeveloperChatMessage(sys.Content[0].Text),
+                _ => m
+            }).ToList();
+        }
+
         ClientResult<ChatCompletion> cc = await _chatClient.CompleteChatAsync(messages, options, cancellationToken);
         ChatCompletion delta = cc.Value;
         return new ChatSegment
@@ -63,4 +75,37 @@ public override async Task<ChatSegment> Chat(IReadOnlyList<ChatMessage> messages
             } : null,
         };
     }
+
+    private class DeveloperChatMessage(string content) : SystemChatMessage(content), IJsonModel<DeveloperChatMessage>
+    {
+        public DeveloperChatMessage Create(ref Utf8JsonReader reader, ModelReaderWriterOptions options)
+        {
+            throw new NotImplementedException();
+        }
+
+        public DeveloperChatMessage Create(BinaryData data, ModelReaderWriterOptions options)
+        {
+            throw new NotImplementedException();
+        }
+
+        public string GetFormatFromOptions(ModelReaderWriterOptions options)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Write(Utf8JsonWriter writer, ModelReaderWriterOptions options)
+        {
+            writer.WriteStartObject();
+            writer.WritePropertyName("role"u8);
+            writer.WriteStringValue("developer");
+            writer.WritePropertyName("content"u8);
+            writer.WriteStringValue(Content[0].Text);
+            writer.WriteEndObject();
+        }
+
+        public BinaryData Write(ModelReaderWriterOptions options)
+        {
+            throw new NotImplementedException();
+        }
+    }
 }