OAI: Add stream to gen params

Good for logging. Signed-off-by: kingbri <[email protected]>
theroyallab · Mar 21, 2024 · 56fdfb5 · 56fdfb5
1 parent 69e41e9
commit 56fdfb5
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/backends/exllamav2/model.py b/backends/exllamav2/model.py
@@ -863,6 +863,7 @@ def generate_gen_sync(
         # Some options are too large, so log the args instead
         log_generation_params(
             max_tokens=max_tokens,
+            stream=kwargs.get("stream"),
             **gen_settings_log_dict,
             token_healing=token_healing,
             auto_scale_penalty_range=auto_scale_penalty_range,

diff --git a/endpoints/OAI/types/common.py b/endpoints/OAI/types/common.py
@@ -43,6 +43,9 @@ class CommonCompletionRequest(BaseSamplerRequest):
     )
 
     def to_gen_params(self):
-        extra_gen_params = {"logprobs": self.logprobs}
+        extra_gen_params = {
+            "stream": self.stream,
+            "logprobs": self.logprobs,
+        }
 
         return super().to_gen_params(**extra_gen_params)