Properly count used tokens

StampyAI · Dec 2, 2023 · 9831aea · 9831aea
1 parent 51d82c8
commit 9831aea
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/api/src/stampy_chat/settings.py b/api/src/stampy_chat/settings.py
@@ -173,4 +173,10 @@ def history_tokens(self):
 
     @property
     def max_response_tokens(self):
-        return min(self.maxNumTokens - self.context_tokens - self.history_tokens, self.maxCompletionTokens)
+        available_tokens = (
+            self.maxNumTokens -
+            self.context_tokens - len(self.encoder.encode(self.context_prompt)) -
+            self.history_tokens - len(self.encoder.encode(self.history_prompt)) -
+            len(self.encoder.encode(self.question_prompt))
+        )
+        return min(available_tokens, self.maxCompletionTokens)