Model: Add BOS token to prompt logs

If add_bos_token is enabled, the BOS token gets appended to the logged prompt if logging is enabled. Signed-off-by: kingbri <[email protected]>
theroyallab · Aug 30, 2024 · 10d9419 · 10d9419
1 parent 96fce34
commit 10d9419
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/backends/exllamav2/model.py b/backends/exllamav2/model.py
@@ -1163,8 +1163,12 @@ async def generate_gen(
         # This is an inverse of skip_special_tokens
         decode_special_tokens = unwrap(not kwargs.get("skip_special_tokens"), False)
 
-        # Log prompt to console
-        log_prompt(prompt, request_id, negative_prompt)
+        # Log prompt to console. Add the BOS token if specified
+        log_prompt(
+            f"{self.tokenizer.bos_token if add_bos_token else ''}{prompt}",
+            request_id,
+            negative_prompt
+        )
 
         # Create and add a new job
         # Don't use the request ID here as there can be multiple jobs per request