simplify tokenizer creation logic

sdcb · Dec 9, 2024 · b00c2ca · b00c2ca
1 parent 5483024
commit b00c2ca
Showing 1 changed file with 3 additions and 10 deletions.
diff --git a/src/BE/Services/Conversations/ConversationService.cs b/src/BE/Services/Conversations/ConversationService.cs
@@ -19,18 +19,11 @@ public abstract partial class ConversationService : IDisposable
     public ConversationService(Model model)
     {
         Model = model;
-        try
+        if (model.ModelReference.Tokenizer is not null)
         {
-            if (model.ModelReference.Tokenizer is not null)
-            {
-                Tokenizer = TiktokenTokenizer.CreateForEncoding(model.ModelReference.Tokenizer.Name);
-            }
-            else
-            {
-                Tokenizer = TiktokenTokenizer.CreateForModel(Model.ModelReference.Name);
-            }
+            Tokenizer = TiktokenTokenizer.CreateForEncoding(model.ModelReference.Tokenizer.Name);
         }
-        catch (NotSupportedException)
+        else
         {
             Tokenizer = TiktokenTokenizer.CreateForEncoding("cl100k_base");
         }