Pass 'model_kwargs for outlines.models.llamacpp as dict (#744)

Fixes [this issue](#743)
dottxt-ai · Mar 14, 2024 · 5c15e8c · 5c15e8c
1 parent 6484d8c
commit 5c15e8c
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/outlines/models/llamacpp.py b/outlines/models/llamacpp.py
@@ -25,6 +25,7 @@ def __init__(self, model: "Llama"):
 def llamacpp(model_path: str, device: Optional[str] = None, **model_kwargs) -> LlamaCpp:
     from llama_cpp import Llama
 
+    model_kwargs = model_kwargs.get("model_kwargs", {})
     if device == "cuda":
         model_kwargs["n_gpu_layers"].setdefault(-1)