From 9831aea2bd8233f974dddf5fe53dc38869c1a297 Mon Sep 17 00:00:00 2001 From: Daniel O'Connell Date: Sat, 2 Dec 2023 20:47:32 +0100 Subject: [PATCH] Properly count used tokens --- api/src/stampy_chat/settings.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/api/src/stampy_chat/settings.py b/api/src/stampy_chat/settings.py index 50d42c4..8f8a737 100644 --- a/api/src/stampy_chat/settings.py +++ b/api/src/stampy_chat/settings.py @@ -173,4 +173,10 @@ def history_tokens(self): @property def max_response_tokens(self): - return min(self.maxNumTokens - self.context_tokens - self.history_tokens, self.maxCompletionTokens) + available_tokens = ( + self.maxNumTokens - + self.context_tokens - len(self.encoder.encode(self.context_prompt)) - + self.history_tokens - len(self.encoder.encode(self.history_prompt)) - + len(self.encoder.encode(self.question_prompt)) + ) + return min(available_tokens, self.maxCompletionTokens)