Merge pull request #134 from StampyAI/better-tokens-count

Properly count used tokens
StampyAI · Dec 3, 2023 · c663939 · c663939
2 parents 51d82c8 + 9831aea
commit c663939
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/api/src/stampy_chat/settings.py b/api/src/stampy_chat/settings.py
@@ -173,4 +173,10 @@ def history_tokens(self):
 
     @property
     def max_response_tokens(self):
-        return min(self.maxNumTokens - self.context_tokens - self.history_tokens, self.maxCompletionTokens)
+        available_tokens = (
+            self.maxNumTokens -
+            self.context_tokens - len(self.encoder.encode(self.context_prompt)) -
+            self.history_tokens - len(self.encoder.encode(self.history_prompt)) -
+            len(self.encoder.encode(self.question_prompt))
+        )
+        return min(available_tokens, self.maxCompletionTokens)